2024-12-08 03:39:48,233 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 03:39:48,242 main DEBUG Took 0.007739 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 03:39:48,243 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 03:39:48,243 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 03:39:48,244 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 03:39:48,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,250 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 03:39:48,261 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,263 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,264 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,265 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,266 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,266 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,267 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,267 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,267 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,268 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,268 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,268 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,269 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,269 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,270 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:39:48,270 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,271 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 03:39:48,272 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:39:48,273 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 03:39:48,274 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 03:39:48,275 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 03:39:48,276 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 03:39:48,276 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 03:39:48,283 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 03:39:48,285 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 03:39:48,287 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 03:39:48,287 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 03:39:48,287 main DEBUG createAppenders(={Console}) 2024-12-08 03:39:48,288 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-08 03:39:48,288 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 03:39:48,289 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-08 03:39:48,289 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 03:39:48,289 main DEBUG OutputStream closed 2024-12-08 03:39:48,290 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 03:39:48,290 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 03:39:48,290 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-08 03:39:48,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 03:39:48,357 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 03:39:48,358 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 03:39:48,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 03:39:48,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 03:39:48,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 03:39:48,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 03:39:48,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 03:39:48,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 03:39:48,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 03:39:48,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 03:39:48,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 03:39:48,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 03:39:48,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 03:39:48,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 03:39:48,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 03:39:48,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 03:39:48,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 03:39:48,366 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 03:39:48,366 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-08 03:39:48,367 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 03:39:48,367 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-08T03:39:48,597 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f 2024-12-08 03:39:48,600 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 03:39:48,600 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T03:39:48,608 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-08T03:39:48,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T03:39:48,647 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6, deleteOnExit=true 2024-12-08T03:39:48,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T03:39:48,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/test.cache.data in system properties and HBase conf 2024-12-08T03:39:48,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T03:39:48,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/hadoop.log.dir in system properties and HBase conf 2024-12-08T03:39:48,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T03:39:48,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T03:39:48,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T03:39:48,738 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T03:39:48,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T03:39:48,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T03:39:48,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T03:39:48,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T03:39:48,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T03:39:48,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T03:39:48,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T03:39:48,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T03:39:48,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T03:39:48,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T03:39:48,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/nfs.dump.dir in system properties and HBase conf 2024-12-08T03:39:48,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/java.io.tmpdir in system properties and HBase conf 2024-12-08T03:39:48,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T03:39:48,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T03:39:48,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T03:39:49,786 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T03:39:49,882 INFO [Time-limited test {}] log.Log(170): Logging initialized @2301ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T03:39:49,967 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T03:39:50,042 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T03:39:50,069 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T03:39:50,069 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T03:39:50,071 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T03:39:50,084 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T03:39:50,088 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/hadoop.log.dir/,AVAILABLE} 2024-12-08T03:39:50,090 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T03:39:50,322 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/java.io.tmpdir/jetty-localhost-42791-hadoop-hdfs-3_4_1-tests_jar-_-any-11031889538520430773/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T03:39:50,329 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42791} 2024-12-08T03:39:50,329 INFO [Time-limited test {}] server.Server(415): Started @2749ms 2024-12-08T03:39:51,128 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T03:39:51,135 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T03:39:51,137 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T03:39:51,137 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T03:39:51,137 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T03:39:51,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/hadoop.log.dir/,AVAILABLE} 2024-12-08T03:39:51,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T03:39:51,240 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/java.io.tmpdir/jetty-localhost-39835-hadoop-hdfs-3_4_1-tests_jar-_-any-7670528579087949236/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T03:39:51,241 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39835} 2024-12-08T03:39:51,242 INFO [Time-limited test {}] server.Server(415): Started @3661ms 2024-12-08T03:39:51,298 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T03:39:52,194 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/data/data1/current/BP-1611339395-172.17.0.2-1733629189334/current, will proceed with Du for space computation calculation, 2024-12-08T03:39:52,194 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/data/data2/current/BP-1611339395-172.17.0.2-1733629189334/current, will proceed with Du for space computation calculation, 2024-12-08T03:39:52,239 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T03:39:52,298 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8201e6b02df96559 with lease ID 0x86eaefd2a0765ab4: Processing first storage report for DS-4bb07580-50da-48a7-adf1-1c56d2e315f8 from datanode DatanodeRegistration(127.0.0.1:38303, datanodeUuid=cea92d07-14e2-464c-89b3-db1f91e631d3, infoPort=41763, infoSecurePort=0, ipcPort=42699, storageInfo=lv=-57;cid=testClusterID;nsid=309558271;c=1733629189334) 2024-12-08T03:39:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8201e6b02df96559 with lease ID 0x86eaefd2a0765ab4: from storage DS-4bb07580-50da-48a7-adf1-1c56d2e315f8 node DatanodeRegistration(127.0.0.1:38303, datanodeUuid=cea92d07-14e2-464c-89b3-db1f91e631d3, infoPort=41763, infoSecurePort=0, ipcPort=42699, storageInfo=lv=-57;cid=testClusterID;nsid=309558271;c=1733629189334), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T03:39:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8201e6b02df96559 with lease ID 0x86eaefd2a0765ab4: Processing first storage report for DS-b70a9b4b-15da-477c-9e68-da573bc707c4 from datanode DatanodeRegistration(127.0.0.1:38303, datanodeUuid=cea92d07-14e2-464c-89b3-db1f91e631d3, infoPort=41763, infoSecurePort=0, ipcPort=42699, storageInfo=lv=-57;cid=testClusterID;nsid=309558271;c=1733629189334) 2024-12-08T03:39:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8201e6b02df96559 with lease ID 0x86eaefd2a0765ab4: from storage DS-b70a9b4b-15da-477c-9e68-da573bc707c4 node DatanodeRegistration(127.0.0.1:38303, datanodeUuid=cea92d07-14e2-464c-89b3-db1f91e631d3, infoPort=41763, infoSecurePort=0, ipcPort=42699, storageInfo=lv=-57;cid=testClusterID;nsid=309558271;c=1733629189334), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T03:39:52,369 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f 2024-12-08T03:39:52,487 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/zookeeper_0, clientPort=53034, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T03:39:52,499 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53034 2024-12-08T03:39:52,507 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:52,510 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:52,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741825_1001 (size=7) 2024-12-08T03:39:53,103 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a with version=8 2024-12-08T03:39:53,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/hbase-staging 2024-12-08T03:39:53,194 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T03:39:53,429 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T03:39:53,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:39:53,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T03:39:53,447 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T03:39:53,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:39:53,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T03:39:53,615 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T03:39:53,687 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T03:39:53,699 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T03:39:53,704 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T03:39:53,741 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2490 (auto-detected) 2024-12-08T03:39:53,743 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T03:39:53,769 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46215 2024-12-08T03:39:53,793 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46215 connecting to ZooKeeper ensemble=127.0.0.1:53034 2024-12-08T03:39:53,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462150x0, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T03:39:53,929 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46215-0x100038fe7ae0000 connected 2024-12-08T03:39:54,010 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:54,013 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:54,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T03:39:54,030 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a, hbase.cluster.distributed=false 2024-12-08T03:39:54,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T03:39:54,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46215 2024-12-08T03:39:54,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46215 2024-12-08T03:39:54,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46215 2024-12-08T03:39:54,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46215 2024-12-08T03:39:54,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46215 2024-12-08T03:39:54,210 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T03:39:54,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:39:54,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T03:39:54,213 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T03:39:54,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:39:54,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T03:39:54,216 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T03:39:54,219 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T03:39:54,222 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42629 2024-12-08T03:39:54,225 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42629 connecting to ZooKeeper ensemble=127.0.0.1:53034 2024-12-08T03:39:54,227 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:54,231 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:54,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426290x0, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T03:39:54,258 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426290x0, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T03:39:54,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42629-0x100038fe7ae0001 connected 2024-12-08T03:39:54,263 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T03:39:54,274 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T03:39:54,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T03:39:54,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T03:39:54,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42629 2024-12-08T03:39:54,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42629 2024-12-08T03:39:54,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42629 2024-12-08T03:39:54,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42629 2024-12-08T03:39:54,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42629 2024-12-08T03:39:54,307 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32df4b4bb439:46215 2024-12-08T03:39:54,309 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32df4b4bb439,46215,1733629193261 2024-12-08T03:39:54,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:39:54,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:39:54,342 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32df4b4bb439,46215,1733629193261 2024-12-08T03:39:54,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:54,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T03:39:54,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:54,376 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T03:39:54,378 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32df4b4bb439,46215,1733629193261 from backup master directory 2024-12-08T03:39:54,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32df4b4bb439,46215,1733629193261 2024-12-08T03:39:54,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:39:54,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:39:54,392 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T03:39:54,392 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32df4b4bb439,46215,1733629193261 2024-12-08T03:39:54,394 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T03:39:54,398 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T03:39:54,456 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/hbase.id] with ID: 683332fe-025f-4451-984f-68618f51f751 2024-12-08T03:39:54,456 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/.tmp/hbase.id 2024-12-08T03:39:54,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741826_1002 (size=42) 2024-12-08T03:39:54,469 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/.tmp/hbase.id]:[hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/hbase.id] 2024-12-08T03:39:54,518 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:39:54,524 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T03:39:54,545 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-08T03:39:54,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:54,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:54,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741827_1003 (size=196) 2024-12-08T03:39:54,605 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:39:54,607 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T03:39:54,621 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:54,625 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T03:39:54,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741828_1004 (size=1189) 2024-12-08T03:39:54,668 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store 2024-12-08T03:39:54,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741829_1005 (size=34) 2024-12-08T03:39:54,695 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T03:39:54,698 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:39:54,699 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T03:39:54,700 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:39:54,700 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:39:54,701 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T03:39:54,701 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:39:54,702 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:39:54,703 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733629194699Disabling compacts and flushes for region at 1733629194699Disabling writes for close at 1733629194701 (+2 ms)Writing region close event to WAL at 1733629194702 (+1 ms)Closed at 1733629194702 2024-12-08T03:39:54,705 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/.initializing 2024-12-08T03:39:54,705 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/WALs/32df4b4bb439,46215,1733629193261 2024-12-08T03:39:54,714 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T03:39:54,731 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C46215%2C1733629193261, suffix=, logDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/WALs/32df4b4bb439,46215,1733629193261, archiveDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/oldWALs, maxLogs=10 2024-12-08T03:39:54,756 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/WALs/32df4b4bb439,46215,1733629193261/32df4b4bb439%2C46215%2C1733629193261.1733629194736, exclude list is [], retry=0 2024-12-08T03:39:54,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38303,DS-4bb07580-50da-48a7-adf1-1c56d2e315f8,DISK] 2024-12-08T03:39:54,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T03:39:54,816 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/WALs/32df4b4bb439,46215,1733629193261/32df4b4bb439%2C46215%2C1733629193261.1733629194736 2024-12-08T03:39:54,817 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41763:41763)] 2024-12-08T03:39:54,817 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:39:54,818 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:39:54,821 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,822 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T03:39:54,893 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:54,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:54,896 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T03:39:54,899 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:54,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:39:54,901 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,903 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T03:39:54,903 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:54,904 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:39:54,904 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T03:39:54,907 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:54,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:39:54,909 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,912 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,913 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,919 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,919 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,923 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T03:39:54,927 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:39:54,932 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:39:54,933 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68420625, jitterRate=0.019546762108802795}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T03:39:54,938 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733629194835Initializing all the Stores at 1733629194837 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629194838 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629194839 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629194839Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629194839Cleaning up temporary data from old regions at 1733629194919 (+80 ms)Region opened successfully at 1733629194938 (+19 ms) 2024-12-08T03:39:54,939 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T03:39:54,975 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbfe831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T03:39:55,003 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T03:39:55,012 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T03:39:55,012 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T03:39:55,015 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T03:39:55,016 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T03:39:55,020 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-08T03:39:55,020 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T03:39:55,047 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T03:39:55,057 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T03:39:55,086 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T03:39:55,090 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T03:39:55,092 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T03:39:55,107 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T03:39:55,110 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T03:39:55,114 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T03:39:55,124 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T03:39:55,125 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T03:39:55,132 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T03:39:55,149 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T03:39:55,157 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T03:39:55,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T03:39:55,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T03:39:55,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:55,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:55,168 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32df4b4bb439,46215,1733629193261, sessionid=0x100038fe7ae0000, setting cluster-up flag (Was=false) 2024-12-08T03:39:55,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:55,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:55,262 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T03:39:55,267 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,46215,1733629193261 2024-12-08T03:39:55,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:55,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:55,315 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T03:39:55,317 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,46215,1733629193261 2024-12-08T03:39:55,323 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T03:39:55,391 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T03:39:55,399 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(746): ClusterId : 683332fe-025f-4451-984f-68618f51f751 2024-12-08T03:39:55,400 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T03:39:55,401 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T03:39:55,406 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T03:39:55,411 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32df4b4bb439,46215,1733629193261 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T03:39:55,433 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T03:39:55,433 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T03:39:55,434 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:39:55,434 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:39:55,435 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:39:55,435 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:39:55,435 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32df4b4bb439:0, corePoolSize=10, maxPoolSize=10 2024-12-08T03:39:55,436 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,436 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T03:39:55,436 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733629225439 2024-12-08T03:39:55,441 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T03:39:55,442 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T03:39:55,442 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T03:39:55,443 DEBUG [RS:0;32df4b4bb439:42629 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd41370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T03:39:55,443 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T03:39:55,443 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T03:39:55,447 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T03:39:55,448 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T03:39:55,449 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T03:39:55,449 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T03:39:55,450 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:55,451 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T03:39:55,453 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,460 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T03:39:55,461 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T03:39:55,462 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T03:39:55,470 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32df4b4bb439:42629 2024-12-08T03:39:55,471 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T03:39:55,471 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T03:39:55,474 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T03:39:55,475 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T03:39:55,475 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T03:39:55,478 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733629195473,5,FailOnTimeoutGroup] 2024-12-08T03:39:55,479 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,46215,1733629193261 with port=42629, startcode=1733629194165 2024-12-08T03:39:55,482 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733629195478,5,FailOnTimeoutGroup] 2024-12-08T03:39:55,482 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,482 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T03:39:55,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741831_1007 (size=1321) 2024-12-08T03:39:55,484 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,485 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,488 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T03:39:55,489 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:39:55,496 DEBUG [RS:0;32df4b4bb439:42629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T03:39:55,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741832_1008 (size=32) 2024-12-08T03:39:55,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32911, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T03:39:55,541 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,42629,1733629194165 2024-12-08T03:39:55,544 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46215 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:55,563 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:39:55,563 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41611 2024-12-08T03:39:55,563 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T03:39:55,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T03:39:55,666 DEBUG [RS:0;32df4b4bb439:42629 {}] zookeeper.ZKUtil(111): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,42629,1733629194165 2024-12-08T03:39:55,666 WARN [RS:0;32df4b4bb439:42629 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T03:39:55,666 INFO [RS:0;32df4b4bb439:42629 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T03:39:55,667 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165 2024-12-08T03:39:55,668 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,42629,1733629194165] 2024-12-08T03:39:55,695 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T03:39:55,711 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T03:39:55,718 INFO [RS:0;32df4b4bb439:42629 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T03:39:55,719 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,720 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T03:39:55,727 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T03:39:55,728 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,729 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,729 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,729 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,729 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,730 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:39:55,731 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T03:39:55,731 DEBUG [RS:0;32df4b4bb439:42629 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T03:39:55,732 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,732 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,732 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,732 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,732 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,732 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42629,1733629194165-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T03:39:55,758 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T03:39:55,761 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42629,1733629194165-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,761 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,761 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.Replication(171): 32df4b4bb439,42629,1733629194165 started 2024-12-08T03:39:55,784 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:55,785 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,42629,1733629194165, RpcServer on 32df4b4bb439/172.17.0.2:42629, sessionid=0x100038fe7ae0001 2024-12-08T03:39:55,786 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T03:39:55,786 DEBUG [RS:0;32df4b4bb439:42629 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,42629,1733629194165 2024-12-08T03:39:55,786 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,42629,1733629194165' 2024-12-08T03:39:55,786 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T03:39:55,788 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T03:39:55,788 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T03:39:55,789 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T03:39:55,789 DEBUG [RS:0;32df4b4bb439:42629 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,42629,1733629194165 2024-12-08T03:39:55,789 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,42629,1733629194165' 2024-12-08T03:39:55,789 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T03:39:55,790 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T03:39:55,790 DEBUG [RS:0;32df4b4bb439:42629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T03:39:55,790 INFO [RS:0;32df4b4bb439:42629 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T03:39:55,790 INFO [RS:0;32df4b4bb439:42629 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T03:39:55,900 INFO [RS:0;32df4b4bb439:42629 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T03:39:55,905 INFO [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C42629%2C1733629194165, suffix=, logDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165, archiveDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/oldWALs, maxLogs=32 2024-12-08T03:39:55,909 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:39:55,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T03:39:55,915 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T03:39:55,915 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:55,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:55,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T03:39:55,920 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T03:39:55,920 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:55,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:55,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T03:39:55,924 DEBUG [RS:0;32df4b4bb439:42629 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165/32df4b4bb439%2C42629%2C1733629194165.1733629195908, exclude list is [], retry=0 2024-12-08T03:39:55,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T03:39:55,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:55,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:55,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T03:39:55,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38303,DS-4bb07580-50da-48a7-adf1-1c56d2e315f8,DISK] 2024-12-08T03:39:55,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T03:39:55,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:55,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:55,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T03:39:55,932 INFO [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165/32df4b4bb439%2C42629%2C1733629194165.1733629195908 2024-12-08T03:39:55,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740 2024-12-08T03:39:55,933 DEBUG [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41763:41763)] 2024-12-08T03:39:55,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740 2024-12-08T03:39:55,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T03:39:55,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T03:39:55,939 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:39:55,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T03:39:55,946 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:39:55,948 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70603246, jitterRate=0.05207034945487976}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:39:55,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733629195909Initializing all the Stores at 1733629195911 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629195911Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629195911Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629195911Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629195911Cleaning up temporary data from old regions at 1733629195937 (+26 ms)Region opened successfully at 1733629195949 (+12 ms) 2024-12-08T03:39:55,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T03:39:55,950 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T03:39:55,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T03:39:55,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T03:39:55,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T03:39:55,951 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T03:39:55,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733629195949Disabling compacts and flushes for region at 1733629195949Disabling writes for close at 1733629195950 (+1 ms)Writing region close event to WAL at 1733629195951 (+1 ms)Closed at 1733629195951 2024-12-08T03:39:55,954 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T03:39:55,954 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T03:39:55,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T03:39:55,966 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T03:39:55,968 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T03:39:56,122 DEBUG [32df4b4bb439:46215 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T03:39:56,134 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:56,140 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,42629,1733629194165, state=OPENING 2024-12-08T03:39:56,157 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T03:39:56,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:56,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:39:56,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:39:56,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:39:56,167 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T03:39:56,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:39:56,344 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T03:39:56,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33015, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T03:39:56,358 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T03:39:56,359 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T03:39:56,360 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T03:39:56,364 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C42629%2C1733629194165.meta, suffix=.meta, logDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165, archiveDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/oldWALs, maxLogs=32 2024-12-08T03:39:56,384 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165/32df4b4bb439%2C42629%2C1733629194165.meta.1733629196366.meta, exclude list is [], retry=0 2024-12-08T03:39:56,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38303,DS-4bb07580-50da-48a7-adf1-1c56d2e315f8,DISK] 2024-12-08T03:39:56,392 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/WALs/32df4b4bb439,42629,1733629194165/32df4b4bb439%2C42629%2C1733629194165.meta.1733629196366.meta 2024-12-08T03:39:56,392 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41763:41763)] 2024-12-08T03:39:56,393 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:39:56,395 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T03:39:56,397 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T03:39:56,402 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T03:39:56,407 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T03:39:56,408 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:39:56,408 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T03:39:56,408 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T03:39:56,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T03:39:56,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T03:39:56,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:56,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:56,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T03:39:56,418 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T03:39:56,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:56,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:56,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T03:39:56,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T03:39:56,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:56,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:56,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T03:39:56,424 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T03:39:56,424 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:56,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:39:56,426 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T03:39:56,427 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740 2024-12-08T03:39:56,430 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740 2024-12-08T03:39:56,432 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T03:39:56,432 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T03:39:56,433 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:39:56,436 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T03:39:56,438 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67472767, jitterRate=0.005422577261924744}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:39:56,438 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T03:39:56,439 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733629196409Writing region info on filesystem at 1733629196409Initializing all the Stores at 1733629196411 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629196411Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629196411Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629196411Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733629196411Cleaning up temporary data from old regions at 1733629196432 (+21 ms)Running coprocessor post-open hooks at 1733629196438 (+6 ms)Region opened successfully at 1733629196439 (+1 ms) 2024-12-08T03:39:56,446 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733629196334 2024-12-08T03:39:56,455 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T03:39:56,455 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T03:39:56,457 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:56,460 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,42629,1733629194165, state=OPEN 2024-12-08T03:39:56,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T03:39:56,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T03:39:56,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:39:56,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:39:56,512 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:56,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T03:39:56,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,42629,1733629194165 in 343 msec 2024-12-08T03:39:56,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T03:39:56,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 560 msec 2024-12-08T03:39:56,532 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T03:39:56,532 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T03:39:56,559 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:56,561 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:56,589 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:56,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58627, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:56,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2620 sec 2024-12-08T03:39:56,617 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733629196617, completionTime=-1 2024-12-08T03:39:56,620 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T03:39:56,620 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T03:39:56,647 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T03:39:56,647 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733629256647 2024-12-08T03:39:56,647 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733629316647 2024-12-08T03:39:56,647 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-08T03:39:56,650 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46215,1733629193261-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,650 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46215,1733629193261-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,650 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46215,1733629193261-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,652 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32df4b4bb439:46215, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,652 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,653 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,660 DEBUG [master/32df4b4bb439:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T03:39:56,693 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.301sec 2024-12-08T03:39:56,694 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T03:39:56,696 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T03:39:56,698 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T03:39:56,699 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T03:39:56,699 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T03:39:56,700 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46215,1733629193261-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T03:39:56,701 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46215,1733629193261-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T03:39:56,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:56,712 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T03:39:56,713 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T03:39:56,713 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T03:39:56,713 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T03:39:56,714 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46215,1733629193261-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:39:56,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:56,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:56,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b270b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:56,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:56,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:56,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:56,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:56,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:56,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32df4b4bb439,46215,1733629193261 2024-12-08T03:39:56,799 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=18335 2024-12-08T03:39:56,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:39:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:39:56,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c59fc64 2024-12-08T03:39:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:39:56,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:39:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:39:56,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:39:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:39:56,878 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:39:56,881 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:56,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-08T03:39:56,885 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:39:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:39:56,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741835_1011 (size=959) 2024-12-08T03:39:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:39:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:39:57,314 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:39:57,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741836_1012 (size=53) 2024-12-08T03:39:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:39:57,724 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:39:57,724 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9b6311f5dfb022964ba60e0ab8e515b0, disabling compactions & flushes 2024-12-08T03:39:57,724 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:57,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:57,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. after waiting 0 ms 2024-12-08T03:39:57,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:57,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:57,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9b6311f5dfb022964ba60e0ab8e515b0: Waiting for close lock at 1733629197724Disabling compacts and flushes for region at 1733629197724Disabling writes for close at 1733629197725 (+1 ms)Writing region close event to WAL at 1733629197725Closed at 1733629197725 2024-12-08T03:39:57,728 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:39:57,734 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733629197728"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733629197728"}]},"ts":"1733629197728"} 2024-12-08T03:39:57,740 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:39:57,742 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:39:57,746 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629197743"}]},"ts":"1733629197743"} 2024-12-08T03:39:57,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:39:57,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, ASSIGN}] 2024-12-08T03:39:57,757 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, ASSIGN 2024-12-08T03:39:57,759 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, ASSIGN; state=OFFLINE, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=false 2024-12-08T03:39:57,910 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9b6311f5dfb022964ba60e0ab8e515b0, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:57,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, ASSIGN because future has completed 2024-12-08T03:39:57,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:39:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:39:58,080 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:58,081 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:39:58,081 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,081 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:39:58,081 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,081 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,084 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,086 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b6311f5dfb022964ba60e0ab8e515b0 columnFamilyName A 2024-12-08T03:39:58,087 DEBUG [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:58,088 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] regionserver.HStore(327): Store=9b6311f5dfb022964ba60e0ab8e515b0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:39:58,088 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,091 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b6311f5dfb022964ba60e0ab8e515b0 columnFamilyName B 2024-12-08T03:39:58,091 DEBUG [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:58,093 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] regionserver.HStore(327): Store=9b6311f5dfb022964ba60e0ab8e515b0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:39:58,093 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,096 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b6311f5dfb022964ba60e0ab8e515b0 columnFamilyName C 2024-12-08T03:39:58,096 DEBUG [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:39:58,097 INFO [StoreOpener-9b6311f5dfb022964ba60e0ab8e515b0-1 {}] regionserver.HStore(327): Store=9b6311f5dfb022964ba60e0ab8e515b0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:39:58,097 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,099 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,100 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,102 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,102 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,103 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:39:58,106 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,110 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:39:58,112 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9b6311f5dfb022964ba60e0ab8e515b0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74224965, jitterRate=0.1060381680727005}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:39:58,112 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:58,113 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9b6311f5dfb022964ba60e0ab8e515b0: Running coprocessor pre-open hook at 1733629198082Writing region info on filesystem at 1733629198082Initializing all the Stores at 1733629198083 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629198083Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629198083Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629198083Cleaning up temporary data from old regions at 1733629198102 (+19 ms)Running coprocessor post-open hooks at 1733629198112 (+10 ms)Region opened successfully at 1733629198113 (+1 ms) 2024-12-08T03:39:58,116 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., pid=6, masterSystemTime=1733629198070 2024-12-08T03:39:58,120 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:58,121 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:58,122 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9b6311f5dfb022964ba60e0ab8e515b0, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:58,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:39:58,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T03:39:58,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 in 214 msec 2024-12-08T03:39:58,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T03:39:58,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, ASSIGN in 380 msec 2024-12-08T03:39:58,141 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:39:58,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629198141"}]},"ts":"1733629198141"} 2024-12-08T03:39:58,145 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:39:58,147 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:39:58,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2860 sec 2024-12-08T03:39:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:39:59,040 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:39:59,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1cdbfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@972c14f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,053 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,060 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:39:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:39:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1432456c 2024-12-08T03:39:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:39:59,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:39:59,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,071 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,074 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,078 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,081 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,084 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,094 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,097 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f89bd3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588e13f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60075f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,110 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19664801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,117 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7489f107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,120 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333dd90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,124 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d00fff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,126 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:39:59,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:39:59,130 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:39:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:39:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:39:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2341aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:39:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:39:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:39:59,134 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:39:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fb5a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:39:59,140 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,140 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,140 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,140 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,140 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,140 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,148 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,150 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,151 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:39:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:39:59,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:39:59,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:39:59,156 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:39:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-08T03:39:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:39:59,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:39:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:39:59,175 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:39:59,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:39:59,239 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:39:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:39:59,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:39:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:39:59,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:39:59,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:39:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/28ebf1387033438e94f0558e571e1781 is 50, key is test_row_0/A:col10/1733629199213/Put/seqid=0 2024-12-08T03:39:59,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741837_1013 (size=14341) 2024-12-08T03:39:59,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/28ebf1387033438e94f0558e571e1781 2024-12-08T03:39:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:39:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629259490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629259484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629259500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629259507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629259508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:39:59,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:39:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/aa967572abbb4a6e9b698270ace56df8 is 50, key is test_row_0/B:col10/1733629199213/Put/seqid=0 2024-12-08T03:39:59,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741838_1014 (size=12001) 2024-12-08T03:39:59,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629259661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629259661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629259662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629259662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629259659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:39:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:39:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:39:59,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629259869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629259879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629259879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:39:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629259880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629259879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:39:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:39:59,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:39:59,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:39:59,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:39:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:39:59,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:39:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/aa967572abbb4a6e9b698270ace56df8 2024-12-08T03:40:00,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/623af4aad8e6422eaa490d491169da61 is 50, key is test_row_0/C:col10/1733629199213/Put/seqid=0 2024-12-08T03:40:00,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741839_1015 (size=12001) 2024-12-08T03:40:00,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/623af4aad8e6422eaa490d491169da61 2024-12-08T03:40:00,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/28ebf1387033438e94f0558e571e1781 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/28ebf1387033438e94f0558e571e1781 2024-12-08T03:40:00,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/28ebf1387033438e94f0558e571e1781, entries=200, sequenceid=14, filesize=14.0 K 2024-12-08T03:40:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/aa967572abbb4a6e9b698270ace56df8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/aa967572abbb4a6e9b698270ace56df8 2024-12-08T03:40:00,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/aa967572abbb4a6e9b698270ace56df8, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T03:40:00,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/623af4aad8e6422eaa490d491169da61 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/623af4aad8e6422eaa490d491169da61 2024-12-08T03:40:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629260190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/623af4aad8e6422eaa490d491169da61, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T03:40:00,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629260190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 9b6311f5dfb022964ba60e0ab8e515b0 in 967ms, sequenceid=14, compaction requested=false 2024-12-08T03:40:00,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,222 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:40:00,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:00,247 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:00,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:40:00,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/38703505d42340cbbfb5d010c139137e is 50, key is test_row_0/A:col10/1733629200241/Put/seqid=0 2024-12-08T03:40:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:40:00,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741840_1016 (size=14341) 2024-12-08T03:40:00,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/38703505d42340cbbfb5d010c139137e 2024-12-08T03:40:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629260289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629260314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629260321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/fae4d1dfe5c34787bb59187691fa5a1c is 50, key is test_row_0/B:col10/1733629200241/Put/seqid=0 2024-12-08T03:40:00,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741841_1017 (size=12001) 2024-12-08T03:40:00,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/fae4d1dfe5c34787bb59187691fa5a1c 2024-12-08T03:40:00,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629260430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629260431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/dde156b9767541b2a5a246b152f0e850 is 50, key is test_row_0/C:col10/1733629200241/Put/seqid=0 2024-12-08T03:40:00,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629260449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741842_1018 (size=12001) 2024-12-08T03:40:00,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629260652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629260652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629260679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629260720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629260719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:00,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:00,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:00,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/dde156b9767541b2a5a246b152f0e850 2024-12-08T03:40:00,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/38703505d42340cbbfb5d010c139137e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/38703505d42340cbbfb5d010c139137e 2024-12-08T03:40:00,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/38703505d42340cbbfb5d010c139137e, entries=200, sequenceid=40, filesize=14.0 K 2024-12-08T03:40:00,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/fae4d1dfe5c34787bb59187691fa5a1c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/fae4d1dfe5c34787bb59187691fa5a1c 2024-12-08T03:40:00,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/fae4d1dfe5c34787bb59187691fa5a1c, entries=150, sequenceid=40, filesize=11.7 K 2024-12-08T03:40:00,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/dde156b9767541b2a5a246b152f0e850 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dde156b9767541b2a5a246b152f0e850 2024-12-08T03:40:00,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629260969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629260969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:00,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dde156b9767541b2a5a246b152f0e850, entries=150, sequenceid=40, filesize=11.7 K 2024-12-08T03:40:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:00,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9b6311f5dfb022964ba60e0ab8e515b0 in 740ms, sequenceid=40, compaction requested=false 2024-12-08T03:40:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:01,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:40:01,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:01,004 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-08T03:40:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:01,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:01,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/585fc22ada0a4dd8aff613d3e5d81f28 is 50, key is test_row_0/A:col10/1733629200305/Put/seqid=0 2024-12-08T03:40:01,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741843_1019 (size=9657) 2024-12-08T03:40:01,036 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/585fc22ada0a4dd8aff613d3e5d81f28 2024-12-08T03:40:01,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/87c53af07e6b4f6e8c1b46de9b9634ce is 50, key is test_row_0/B:col10/1733629200305/Put/seqid=0 2024-12-08T03:40:01,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741844_1020 (size=9657) 2024-12-08T03:40:01,096 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/87c53af07e6b4f6e8c1b46de9b9634ce 2024-12-08T03:40:01,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/8bc43cd3160645b99cab0c324301efc6 is 50, key is test_row_0/C:col10/1733629200305/Put/seqid=0 2024-12-08T03:40:01,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741845_1021 (size=9657) 2024-12-08T03:40:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:40:01,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629261366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629261479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629261488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629261499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,534 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/8bc43cd3160645b99cab0c324301efc6 2024-12-08T03:40:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/585fc22ada0a4dd8aff613d3e5d81f28 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/585fc22ada0a4dd8aff613d3e5d81f28 2024-12-08T03:40:01,569 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/585fc22ada0a4dd8aff613d3e5d81f28, entries=100, sequenceid=50, filesize=9.4 K 2024-12-08T03:40:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/87c53af07e6b4f6e8c1b46de9b9634ce as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/87c53af07e6b4f6e8c1b46de9b9634ce 2024-12-08T03:40:01,594 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/87c53af07e6b4f6e8c1b46de9b9634ce, entries=100, sequenceid=50, filesize=9.4 K 2024-12-08T03:40:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/8bc43cd3160645b99cab0c324301efc6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/8bc43cd3160645b99cab0c324301efc6 2024-12-08T03:40:01,623 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/8bc43cd3160645b99cab0c324301efc6, entries=100, sequenceid=50, filesize=9.4 K 2024-12-08T03:40:01,625 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9b6311f5dfb022964ba60e0ab8e515b0 in 622ms, sequenceid=50, compaction requested=true 2024-12-08T03:40:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-08T03:40:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-08T03:40:01,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T03:40:01,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4530 sec 2024-12-08T03:40:01,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.4780 sec 2024-12-08T03:40:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:01,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-08T03:40:01,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/ccac79cf9cf54bcab7c1e13893a0982a is 50, key is test_row_0/A:col10/1733629201359/Put/seqid=0 2024-12-08T03:40:01,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741846_1022 (size=12001) 2024-12-08T03:40:01,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/ccac79cf9cf54bcab7c1e13893a0982a 2024-12-08T03:40:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629261759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629261759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629261759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/559cc522733f42dc95beda2ec99496be is 50, key is test_row_0/B:col10/1733629201359/Put/seqid=0 2024-12-08T03:40:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741847_1023 (size=12001) 2024-12-08T03:40:01,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/559cc522733f42dc95beda2ec99496be 2024-12-08T03:40:01,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a13773d32eb24da8ba8e9cd03a681ba1 is 50, key is test_row_0/C:col10/1733629201359/Put/seqid=0 2024-12-08T03:40:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741848_1024 (size=12001) 2024-12-08T03:40:01,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a13773d32eb24da8ba8e9cd03a681ba1 2024-12-08T03:40:01,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629261879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629261879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629261879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:01,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/ccac79cf9cf54bcab7c1e13893a0982a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ccac79cf9cf54bcab7c1e13893a0982a 2024-12-08T03:40:01,920 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:40:01,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ccac79cf9cf54bcab7c1e13893a0982a, entries=150, sequenceid=77, filesize=11.7 K 2024-12-08T03:40:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/559cc522733f42dc95beda2ec99496be as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/559cc522733f42dc95beda2ec99496be 2024-12-08T03:40:01,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/559cc522733f42dc95beda2ec99496be, entries=150, sequenceid=77, filesize=11.7 K 2024-12-08T03:40:01,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a13773d32eb24da8ba8e9cd03a681ba1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a13773d32eb24da8ba8e9cd03a681ba1 2024-12-08T03:40:01,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a13773d32eb24da8ba8e9cd03a681ba1, entries=150, sequenceid=77, filesize=11.7 K 2024-12-08T03:40:01,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 9b6311f5dfb022964ba60e0ab8e515b0 in 303ms, sequenceid=77, compaction requested=true 2024-12-08T03:40:01,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:02,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:02,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:02,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:02,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:02,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:02,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:02,016 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:02,016 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:02,024 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T03:40:02,024 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:02,026 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:02,027 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:02,027 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/aa967572abbb4a6e9b698270ace56df8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/fae4d1dfe5c34787bb59187691fa5a1c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/87c53af07e6b4f6e8c1b46de9b9634ce, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/559cc522733f42dc95beda2ec99496be] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=44.6 K 2024-12-08T03:40:02,030 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting aa967572abbb4a6e9b698270ace56df8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733629199213 2024-12-08T03:40:02,030 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:02,030 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:02,030 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:02,030 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/28ebf1387033438e94f0558e571e1781, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/38703505d42340cbbfb5d010c139137e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/585fc22ada0a4dd8aff613d3e5d81f28, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ccac79cf9cf54bcab7c1e13893a0982a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=49.2 K 2024-12-08T03:40:02,031 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting fae4d1dfe5c34787bb59187691fa5a1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629199487 2024-12-08T03:40:02,031 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ebf1387033438e94f0558e571e1781, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733629199213 2024-12-08T03:40:02,032 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 87c53af07e6b4f6e8c1b46de9b9634ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733629200305 2024-12-08T03:40:02,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38703505d42340cbbfb5d010c139137e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629199484 2024-12-08T03:40:02,033 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 559cc522733f42dc95beda2ec99496be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629201305 2024-12-08T03:40:02,033 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585fc22ada0a4dd8aff613d3e5d81f28, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733629200305 2024-12-08T03:40:02,035 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccac79cf9cf54bcab7c1e13893a0982a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629201305 2024-12-08T03:40:02,082 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#12 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:02,086 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/19ad3467b3e14d1c897af3c5b41801ec is 50, key is test_row_0/A:col10/1733629201359/Put/seqid=0 2024-12-08T03:40:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:02,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:02,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:40:02,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/b75114fa0e9746f5a8ca724b9c26f20c is 50, key is test_row_0/A:col10/1733629201751/Put/seqid=0 2024-12-08T03:40:02,126 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#13 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:02,128 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/cbdf676ff6694c63bcd13198a91d50f0 is 50, key is test_row_0/B:col10/1733629201359/Put/seqid=0 2024-12-08T03:40:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741849_1025 (size=12139) 2024-12-08T03:40:02,147 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/19ad3467b3e14d1c897af3c5b41801ec as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/19ad3467b3e14d1c897af3c5b41801ec 2024-12-08T03:40:02,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741850_1026 (size=14341) 2024-12-08T03:40:02,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/b75114fa0e9746f5a8ca724b9c26f20c 2024-12-08T03:40:02,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741851_1027 (size=12139) 2024-12-08T03:40:02,181 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/cbdf676ff6694c63bcd13198a91d50f0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/cbdf676ff6694c63bcd13198a91d50f0 2024-12-08T03:40:02,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/53790a75fe2f4582b9038912a53735b3 is 50, key is test_row_0/B:col10/1733629201751/Put/seqid=0 2024-12-08T03:40:02,193 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 19ad3467b3e14d1c897af3c5b41801ec(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:02,193 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:02,196 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629202000; duration=0sec 2024-12-08T03:40:02,196 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:02,196 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:02,197 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:02,202 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into cbdf676ff6694c63bcd13198a91d50f0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:02,203 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:02,203 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629202015; duration=0sec 2024-12-08T03:40:02,203 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:02,203 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:02,206 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:02,206 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:02,207 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:02,207 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/623af4aad8e6422eaa490d491169da61, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dde156b9767541b2a5a246b152f0e850, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/8bc43cd3160645b99cab0c324301efc6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a13773d32eb24da8ba8e9cd03a681ba1] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=44.6 K 2024-12-08T03:40:02,208 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 623af4aad8e6422eaa490d491169da61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733629199213 2024-12-08T03:40:02,209 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde156b9767541b2a5a246b152f0e850, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629199487 2024-12-08T03:40:02,210 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc43cd3160645b99cab0c324301efc6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733629200305 2024-12-08T03:40:02,211 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13773d32eb24da8ba8e9cd03a681ba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629201305 2024-12-08T03:40:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629262230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741852_1028 (size=12001) 2024-12-08T03:40:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/53790a75fe2f4582b9038912a53735b3 2024-12-08T03:40:02,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629262234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629262236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,255 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#16 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:02,255 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f9da3e3f9d504fc8b87861a852280a4b is 50, key is test_row_0/C:col10/1733629201359/Put/seqid=0 2024-12-08T03:40:02,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1eb2d7bd385441c49c1c213b462556ac is 50, key is test_row_0/C:col10/1733629201751/Put/seqid=0 2024-12-08T03:40:02,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741853_1029 (size=12139) 2024-12-08T03:40:02,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741854_1030 (size=12001) 2024-12-08T03:40:02,307 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f9da3e3f9d504fc8b87861a852280a4b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f9da3e3f9d504fc8b87861a852280a4b 2024-12-08T03:40:02,323 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into f9da3e3f9d504fc8b87861a852280a4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:02,323 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:02,324 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629202015; duration=0sec 2024-12-08T03:40:02,324 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:02,324 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:02,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629262349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629262349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629262349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629262508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,516 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 9b6311f5dfb022964ba60e0ab8e515b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629262520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,522 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_0' in 9b6311f5dfb022964ba60e0ab8e515b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629262559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629262559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629262559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1eb2d7bd385441c49c1c213b462556ac 2024-12-08T03:40:02,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/b75114fa0e9746f5a8ca724b9c26f20c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b75114fa0e9746f5a8ca724b9c26f20c 2024-12-08T03:40:02,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b75114fa0e9746f5a8ca724b9c26f20c, entries=200, sequenceid=88, filesize=14.0 K 2024-12-08T03:40:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/53790a75fe2f4582b9038912a53735b3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/53790a75fe2f4582b9038912a53735b3 2024-12-08T03:40:02,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/53790a75fe2f4582b9038912a53735b3, entries=150, sequenceid=88, filesize=11.7 K 2024-12-08T03:40:02,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1eb2d7bd385441c49c1c213b462556ac as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1eb2d7bd385441c49c1c213b462556ac 2024-12-08T03:40:02,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1eb2d7bd385441c49c1c213b462556ac, entries=150, sequenceid=88, filesize=11.7 K 2024-12-08T03:40:02,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9b6311f5dfb022964ba60e0ab8e515b0 in 677ms, sequenceid=88, compaction requested=false 2024-12-08T03:40:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:02,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:02,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-08T03:40:02,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/9b99dd61e69b46d28de753ba8c648858 is 50, key is test_row_0/A:col10/1733629202875/Put/seqid=0 2024-12-08T03:40:02,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629262907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629262914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629262914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:02,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741855_1031 (size=16681) 2024-12-08T03:40:02,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/9b99dd61e69b46d28de753ba8c648858 2024-12-08T03:40:02,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/68877633022a4eb9810761e3f041d0c4 is 50, key is test_row_0/B:col10/1733629202875/Put/seqid=0 2024-12-08T03:40:02,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741856_1032 (size=12001) 2024-12-08T03:40:02,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/68877633022a4eb9810761e3f041d0c4 2024-12-08T03:40:03,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f4ba77b57f8d49d79bf07380874b6d2c is 50, key is test_row_0/C:col10/1733629202875/Put/seqid=0 2024-12-08T03:40:03,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629263019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629263029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629263029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741857_1033 (size=12001) 2024-12-08T03:40:03,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f4ba77b57f8d49d79bf07380874b6d2c 2024-12-08T03:40:03,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/9b99dd61e69b46d28de753ba8c648858 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9b99dd61e69b46d28de753ba8c648858 2024-12-08T03:40:03,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9b99dd61e69b46d28de753ba8c648858, entries=250, sequenceid=118, filesize=16.3 K 2024-12-08T03:40:03,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/68877633022a4eb9810761e3f041d0c4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/68877633022a4eb9810761e3f041d0c4 2024-12-08T03:40:03,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/68877633022a4eb9810761e3f041d0c4, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T03:40:03,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f4ba77b57f8d49d79bf07380874b6d2c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4ba77b57f8d49d79bf07380874b6d2c 2024-12-08T03:40:03,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4ba77b57f8d49d79bf07380874b6d2c, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T03:40:03,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 9b6311f5dfb022964ba60e0ab8e515b0 in 263ms, sequenceid=118, compaction requested=true 2024-12-08T03:40:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:03,143 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:03,146 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:03,146 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:03,146 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:03,147 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/19ad3467b3e14d1c897af3c5b41801ec, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b75114fa0e9746f5a8ca724b9c26f20c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9b99dd61e69b46d28de753ba8c648858] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=42.1 K 2024-12-08T03:40:03,148 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19ad3467b3e14d1c897af3c5b41801ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629201305 2024-12-08T03:40:03,149 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75114fa0e9746f5a8ca724b9c26f20c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733629201733 2024-12-08T03:40:03,150 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b99dd61e69b46d28de753ba8c648858, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629202211 2024-12-08T03:40:03,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:03,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:03,155 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:03,162 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:03,163 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:03,163 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:03,163 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/cbdf676ff6694c63bcd13198a91d50f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/53790a75fe2f4582b9038912a53735b3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/68877633022a4eb9810761e3f041d0c4] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=35.3 K 2024-12-08T03:40:03,167 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cbdf676ff6694c63bcd13198a91d50f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629201305 2024-12-08T03:40:03,168 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 53790a75fe2f4582b9038912a53735b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733629201733 2024-12-08T03:40:03,169 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 68877633022a4eb9810761e3f041d0c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629202211 2024-12-08T03:40:03,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:03,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:03,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:03,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:03,188 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:03,189 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/8836a8e36d664953afa0bc3048b3dd94 is 50, key is test_row_0/A:col10/1733629202875/Put/seqid=0 2024-12-08T03:40:03,190 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:03,190 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/b76d67e07710475a9d4aa2f131313308 is 50, key is test_row_0/B:col10/1733629202875/Put/seqid=0 2024-12-08T03:40:03,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741858_1034 (size=12241) 2024-12-08T03:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:03,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:03,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:40:03,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741859_1035 (size=12241) 2024-12-08T03:40:03,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/82eade7a64b342acbc67e0f73447eea9 is 50, key is test_row_0/A:col10/1733629202906/Put/seqid=0 2024-12-08T03:40:03,332 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/b76d67e07710475a9d4aa2f131313308 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/b76d67e07710475a9d4aa2f131313308 2024-12-08T03:40:03,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741860_1036 (size=12051) 2024-12-08T03:40:03,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/82eade7a64b342acbc67e0f73447eea9 2024-12-08T03:40:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:40:03,339 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:03,340 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-08T03:40:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:40:03,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:03,349 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into b76d67e07710475a9d4aa2f131313308(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:03,349 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:03,349 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:03,350 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=13, startTime=1733629203155; duration=0sec 2024-12-08T03:40:03,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:03,350 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:03,350 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:03,350 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:03,354 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:03,355 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:03,355 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:03,355 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f9da3e3f9d504fc8b87861a852280a4b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1eb2d7bd385441c49c1c213b462556ac, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4ba77b57f8d49d79bf07380874b6d2c] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=35.3 K 2024-12-08T03:40:03,357 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f9da3e3f9d504fc8b87861a852280a4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629201305 2024-12-08T03:40:03,358 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb2d7bd385441c49c1c213b462556ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733629201733 2024-12-08T03:40:03,359 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f4ba77b57f8d49d79bf07380874b6d2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629202211 2024-12-08T03:40:03,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/ca5263f92aed4417a2e964b417b408e0 is 50, key is test_row_0/B:col10/1733629202906/Put/seqid=0 2024-12-08T03:40:03,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741861_1037 (size=12051) 2024-12-08T03:40:03,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/ca5263f92aed4417a2e964b417b408e0 2024-12-08T03:40:03,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629263385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629263394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629263397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,407 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#25 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:03,407 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f4e8f11b721741e48b72b2556b21cce6 is 50, key is test_row_0/C:col10/1733629202875/Put/seqid=0 2024-12-08T03:40:03,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4 is 50, key is test_row_0/C:col10/1733629202906/Put/seqid=0 2024-12-08T03:40:03,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741862_1038 (size=12241) 2024-12-08T03:40:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:40:03,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741863_1039 (size=12051) 2024-12-08T03:40:03,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4 2024-12-08T03:40:03,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/82eade7a64b342acbc67e0f73447eea9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/82eade7a64b342acbc67e0f73447eea9 2024-12-08T03:40:03,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T03:40:03,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:03,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:03,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:03,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:03,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:03,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629263508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629263508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/82eade7a64b342acbc67e0f73447eea9, entries=150, sequenceid=129, filesize=11.8 K 2024-12-08T03:40:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629263509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/ca5263f92aed4417a2e964b417b408e0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ca5263f92aed4417a2e964b417b408e0 2024-12-08T03:40:03,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ca5263f92aed4417a2e964b417b408e0, entries=150, sequenceid=129, filesize=11.8 K 2024-12-08T03:40:03,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4 2024-12-08T03:40:03,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4, entries=150, sequenceid=129, filesize=11.8 K 2024-12-08T03:40:03,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9b6311f5dfb022964ba60e0ab8e515b0 in 303ms, sequenceid=129, compaction requested=false 2024-12-08T03:40:03,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:03,659 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/8836a8e36d664953afa0bc3048b3dd94 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8836a8e36d664953afa0bc3048b3dd94 2024-12-08T03:40:03,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T03:40:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:03,661 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:40:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:40:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/c14fdef30ee441b78cbb772d2ff5c58b is 50, key is test_row_0/A:col10/1733629203379/Put/seqid=0 2024-12-08T03:40:03,680 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 8836a8e36d664953afa0bc3048b3dd94(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:03,680 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:03,681 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=13, startTime=1733629203143; duration=0sec 2024-12-08T03:40:03,681 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:03,681 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:03,682 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T03:40:03,683 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T03:40:03,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T03:40:03,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T03:40:03,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T03:40:03,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T03:40:03,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T03:40:03,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T03:40:03,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741864_1040 (size=12151) 2024-12-08T03:40:03,730 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/c14fdef30ee441b78cbb772d2ff5c58b 2024-12-08T03:40:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:03,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629263750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629263758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629263760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4 is 50, key is test_row_0/B:col10/1733629203379/Put/seqid=0 2024-12-08T03:40:03,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741865_1041 (size=12151) 2024-12-08T03:40:03,868 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/f4e8f11b721741e48b72b2556b21cce6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4e8f11b721741e48b72b2556b21cce6 2024-12-08T03:40:03,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629263869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629263878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629263879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:03,886 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into f4e8f11b721741e48b72b2556b21cce6(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:03,886 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:03,886 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=13, startTime=1733629203175; duration=0sec 2024-12-08T03:40:03,887 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:03,887 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:40:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629264079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629264088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629264088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4 2024-12-08T03:40:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1a564ac9d4c24fc38058a3f7a9a9bc1d is 50, key is test_row_0/C:col10/1733629203379/Put/seqid=0 2024-12-08T03:40:04,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741866_1042 (size=12151) 2024-12-08T03:40:04,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629264405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629264405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629264405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:40:04,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629264528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629264529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:04,531 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 9b6311f5dfb022964ba60e0ab8e515b0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,532 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_0' in 9b6311f5dfb022964ba60e0ab8e515b0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:04,650 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1a564ac9d4c24fc38058a3f7a9a9bc1d 2024-12-08T03:40:04,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/c14fdef30ee441b78cbb772d2ff5c58b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c14fdef30ee441b78cbb772d2ff5c58b 2024-12-08T03:40:04,671 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c14fdef30ee441b78cbb772d2ff5c58b, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:40:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4 2024-12-08T03:40:04,684 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:40:04,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1a564ac9d4c24fc38058a3f7a9a9bc1d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1a564ac9d4c24fc38058a3f7a9a9bc1d 2024-12-08T03:40:04,699 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1a564ac9d4c24fc38058a3f7a9a9bc1d, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:40:04,701 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1041ms, sequenceid=155, compaction requested=true 2024-12-08T03:40:04,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:04,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:04,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-08T03:40:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-08T03:40:04,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-08T03:40:04,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3540 sec 2024-12-08T03:40:04,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3690 sec 2024-12-08T03:40:04,921 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:04,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:40:04,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/89ddb2561d7e488ba54e8b0fbc137253 is 50, key is test_row_0/A:col10/1733629203754/Put/seqid=0 2024-12-08T03:40:04,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741867_1043 (size=14541) 2024-12-08T03:40:04,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/89ddb2561d7e488ba54e8b0fbc137253 2024-12-08T03:40:04,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5565c5ea548340a98eb95638f251f72b is 50, key is test_row_0/B:col10/1733629203754/Put/seqid=0 2024-12-08T03:40:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629264998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629265007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629265009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741868_1044 (size=12151) 2024-12-08T03:40:05,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5565c5ea548340a98eb95638f251f72b 2024-12-08T03:40:05,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/894366f13a9c4ab5a210644de5ec3e1c is 50, key is test_row_0/C:col10/1733629203754/Put/seqid=0 2024-12-08T03:40:05,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629265119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629265119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741869_1045 (size=12151) 2024-12-08T03:40:05,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629265129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629265329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629265338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629265348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:40:05,499 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:05,500 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-08T03:40:05,503 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:40:05,505 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:05,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:05,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/894366f13a9c4ab5a210644de5ec3e1c 2024-12-08T03:40:05,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/89ddb2561d7e488ba54e8b0fbc137253 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/89ddb2561d7e488ba54e8b0fbc137253 2024-12-08T03:40:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/89ddb2561d7e488ba54e8b0fbc137253, entries=200, sequenceid=169, filesize=14.2 K 2024-12-08T03:40:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5565c5ea548340a98eb95638f251f72b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5565c5ea548340a98eb95638f251f72b 2024-12-08T03:40:05,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5565c5ea548340a98eb95638f251f72b, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T03:40:05,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/894366f13a9c4ab5a210644de5ec3e1c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/894366f13a9c4ab5a210644de5ec3e1c 2024-12-08T03:40:05,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/894366f13a9c4ab5a210644de5ec3e1c, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T03:40:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:40:05,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9b6311f5dfb022964ba60e0ab8e515b0 in 688ms, sequenceid=169, compaction requested=true 2024-12-08T03:40:05,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:05,610 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:05,611 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:05,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:05,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:05,612 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50984 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:05,612 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:05,612 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,613 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8836a8e36d664953afa0bc3048b3dd94, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/82eade7a64b342acbc67e0f73447eea9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c14fdef30ee441b78cbb772d2ff5c58b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/89ddb2561d7e488ba54e8b0fbc137253] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=49.8 K 2024-12-08T03:40:05,613 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:05,613 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8836a8e36d664953afa0bc3048b3dd94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629202211 2024-12-08T03:40:05,614 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:05,614 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,614 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/b76d67e07710475a9d4aa2f131313308, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ca5263f92aed4417a2e964b417b408e0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5565c5ea548340a98eb95638f251f72b] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=47.5 K 2024-12-08T03:40:05,614 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting b76d67e07710475a9d4aa2f131313308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629202211 2024-12-08T03:40:05,614 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82eade7a64b342acbc67e0f73447eea9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733629202906 2024-12-08T03:40:05,615 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting c14fdef30ee441b78cbb772d2ff5c58b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733629203379 2024-12-08T03:40:05,615 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5263f92aed4417a2e964b417b408e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733629202906 2024-12-08T03:40:05,615 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ddb2561d7e488ba54e8b0fbc137253, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733629203742 2024-12-08T03:40:05,616 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2bf87e1fdb4f45a5dfa7eb26e9d7e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733629203379 2024-12-08T03:40:05,616 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5565c5ea548340a98eb95638f251f72b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733629203742 2024-12-08T03:40:05,631 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:05,632 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/d8eb39c09efa4beabb5a59a1979e89e6 is 50, key is test_row_0/A:col10/1733629203754/Put/seqid=0 2024-12-08T03:40:05,642 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:05,643 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:05,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:40:05,643 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/8fc1094994994ccaab521de403e36f2a is 50, key is test_row_0/B:col10/1733629203754/Put/seqid=0 2024-12-08T03:40:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:05,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:40:05,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:05,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:05,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:05,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/cbfea11f09364211b3cfa8991c6e6a4c is 50, key is test_row_0/A:col10/1733629204983/Put/seqid=0 2024-12-08T03:40:05,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629265681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741870_1046 (size=12527) 2024-12-08T03:40:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629265687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629265687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741871_1047 (size=12527) 2024-12-08T03:40:05,729 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/8fc1094994994ccaab521de403e36f2a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8fc1094994994ccaab521de403e36f2a 2024-12-08T03:40:05,751 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into 8fc1094994994ccaab521de403e36f2a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:05,751 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:05,751 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629205611; duration=0sec 2024-12-08T03:40:05,752 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:05,752 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:05,752 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:05,755 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:05,755 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:05,755 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,755 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4e8f11b721741e48b72b2556b21cce6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1a564ac9d4c24fc38058a3f7a9a9bc1d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/894366f13a9c4ab5a210644de5ec3e1c] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=47.5 K 2024-12-08T03:40:05,756 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e8f11b721741e48b72b2556b21cce6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629202211 2024-12-08T03:40:05,757 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca5a3e20d8c4bffb7ccb77f79c5c9f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733629202906 2024-12-08T03:40:05,757 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a564ac9d4c24fc38058a3f7a9a9bc1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733629203379 2024-12-08T03:40:05,758 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 894366f13a9c4ab5a210644de5ec3e1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733629203742 2024-12-08T03:40:05,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741872_1048 (size=12151) 2024-12-08T03:40:05,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/cbfea11f09364211b3cfa8991c6e6a4c 2024-12-08T03:40:05,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/275f6b3204054b8c88b04a88a1bf316d is 50, key is test_row_0/B:col10/1733629204983/Put/seqid=0 2024-12-08T03:40:05,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629265798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629265799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629265809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:05,813 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:05,814 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/87b897606acc4bf6a8a34310c561a3bd is 50, key is test_row_0/C:col10/1733629203754/Put/seqid=0 2024-12-08T03:40:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:40:05,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:40:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:05,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741873_1049 (size=12151) 2024-12-08T03:40:05,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/275f6b3204054b8c88b04a88a1bf316d 2024-12-08T03:40:05,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c67c689bfa714e5f9e6ea42d462e0b43 is 50, key is test_row_0/C:col10/1733629204983/Put/seqid=0 2024-12-08T03:40:05,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741874_1050 (size=12527) 2024-12-08T03:40:05,909 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/87b897606acc4bf6a8a34310c561a3bd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/87b897606acc4bf6a8a34310c561a3bd 2024-12-08T03:40:05,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741875_1051 (size=12151) 2024-12-08T03:40:05,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c67c689bfa714e5f9e6ea42d462e0b43 2024-12-08T03:40:05,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/cbfea11f09364211b3cfa8991c6e6a4c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cbfea11f09364211b3cfa8991c6e6a4c 2024-12-08T03:40:05,931 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into 87b897606acc4bf6a8a34310c561a3bd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:05,931 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:05,931 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629205612; duration=0sec 2024-12-08T03:40:05,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cbfea11f09364211b3cfa8991c6e6a4c, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:40:05,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/275f6b3204054b8c88b04a88a1bf316d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/275f6b3204054b8c88b04a88a1bf316d 2024-12-08T03:40:05,938 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:05,938 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:05,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/275f6b3204054b8c88b04a88a1bf316d, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:40:05,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c67c689bfa714e5f9e6ea42d462e0b43 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c67c689bfa714e5f9e6ea42d462e0b43 2024-12-08T03:40:05,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c67c689bfa714e5f9e6ea42d462e0b43, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:40:05,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 9b6311f5dfb022964ba60e0ab8e515b0 in 316ms, sequenceid=194, compaction requested=false 2024-12-08T03:40:05,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:05,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:40:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:05,982 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:40:05,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/cfaef9917ee043be867defba7fcff5d5 is 50, key is test_row_0/A:col10/1733629205682/Put/seqid=0 2024-12-08T03:40:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:06,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:06,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741876_1052 (size=12151) 2024-12-08T03:40:06,025 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/cfaef9917ee043be867defba7fcff5d5 2024-12-08T03:40:06,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/e00cce4fec74426392694cd8338db48b is 50, key is test_row_0/B:col10/1733629205682/Put/seqid=0 2024-12-08T03:40:06,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741877_1053 (size=12151) 2024-12-08T03:40:06,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629266074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629266083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629266083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,106 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/d8eb39c09efa4beabb5a59a1979e89e6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/d8eb39c09efa4beabb5a59a1979e89e6 2024-12-08T03:40:06,116 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into d8eb39c09efa4beabb5a59a1979e89e6(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:06,116 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:06,116 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629205610; duration=0sec 2024-12-08T03:40:06,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:06,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:40:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629266189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629266199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629266199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629266398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629266408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629266409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,451 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/e00cce4fec74426392694cd8338db48b 2024-12-08T03:40:06,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6de319a9a16c4852b5d6d0e500d993be is 50, key is test_row_0/C:col10/1733629205682/Put/seqid=0 2024-12-08T03:40:06,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741878_1054 (size=12151) 2024-12-08T03:40:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:40:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629266710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629266719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629266719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:06,923 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6de319a9a16c4852b5d6d0e500d993be 2024-12-08T03:40:06,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/cfaef9917ee043be867defba7fcff5d5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cfaef9917ee043be867defba7fcff5d5 2024-12-08T03:40:06,944 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cfaef9917ee043be867defba7fcff5d5, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T03:40:06,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/e00cce4fec74426392694cd8338db48b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/e00cce4fec74426392694cd8338db48b 2024-12-08T03:40:06,954 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/e00cce4fec74426392694cd8338db48b, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T03:40:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6de319a9a16c4852b5d6d0e500d993be as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6de319a9a16c4852b5d6d0e500d993be 2024-12-08T03:40:06,965 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6de319a9a16c4852b5d6d0e500d993be, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T03:40:06,967 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9b6311f5dfb022964ba60e0ab8e515b0 in 985ms, sequenceid=208, compaction requested=true 2024-12-08T03:40:06,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:06,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:06,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-08T03:40:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-08T03:40:06,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T03:40:06,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-12-08T03:40:06,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4750 sec 2024-12-08T03:40:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:07,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:07,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:40:07,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/59f91c85724c43c99a8fe25e383ad0ef is 50, key is test_row_0/A:col10/1733629206061/Put/seqid=0 2024-12-08T03:40:07,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629267264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629267273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629267282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741879_1055 (size=14541) 2024-12-08T03:40:07,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/59f91c85724c43c99a8fe25e383ad0ef 2024-12-08T03:40:07,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1e26460095e841bd894fc67b73c2697a is 50, key is test_row_0/B:col10/1733629206061/Put/seqid=0 2024-12-08T03:40:07,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741880_1056 (size=12151) 2024-12-08T03:40:07,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629267379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629267388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629267388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629267609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629267609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629267630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:40:07,649 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:07,649 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-08T03:40:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:40:07,653 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:07,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:07,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:40:07,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1e26460095e841bd894fc67b73c2697a 2024-12-08T03:40:07,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a1205c9ce3704b738b8b1e102df73359 is 50, key is test_row_0/C:col10/1733629206061/Put/seqid=0 2024-12-08T03:40:07,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:40:07,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:07,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:07,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:07,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:07,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:07,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741881_1057 (size=12151) 2024-12-08T03:40:07,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a1205c9ce3704b738b8b1e102df73359 2024-12-08T03:40:07,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/59f91c85724c43c99a8fe25e383ad0ef as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/59f91c85724c43c99a8fe25e383ad0ef 2024-12-08T03:40:07,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/59f91c85724c43c99a8fe25e383ad0ef, entries=200, sequenceid=235, filesize=14.2 K 2024-12-08T03:40:07,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1e26460095e841bd894fc67b73c2697a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1e26460095e841bd894fc67b73c2697a 2024-12-08T03:40:07,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1e26460095e841bd894fc67b73c2697a, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:40:07,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a1205c9ce3704b738b8b1e102df73359 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1205c9ce3704b738b8b1e102df73359 2024-12-08T03:40:07,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1205c9ce3704b738b8b1e102df73359, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:40:07,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 9b6311f5dfb022964ba60e0ab8e515b0 in 693ms, sequenceid=235, compaction requested=true 2024-12-08T03:40:07,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:07,917 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:07,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:07,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:07,917 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:07,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:07,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:07,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:07,923 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51370 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:07,924 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:07,924 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:07,924 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/d8eb39c09efa4beabb5a59a1979e89e6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cbfea11f09364211b3cfa8991c6e6a4c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cfaef9917ee043be867defba7fcff5d5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/59f91c85724c43c99a8fe25e383ad0ef] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=50.2 K 2024-12-08T03:40:07,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:07,925 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8eb39c09efa4beabb5a59a1979e89e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733629203742 2024-12-08T03:40:07,925 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:07,925 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:07,925 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:07,925 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8fc1094994994ccaab521de403e36f2a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/275f6b3204054b8c88b04a88a1bf316d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/e00cce4fec74426392694cd8338db48b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1e26460095e841bd894fc67b73c2697a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=47.8 K 2024-12-08T03:40:07,926 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc1094994994ccaab521de403e36f2a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733629203742 2024-12-08T03:40:07,926 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbfea11f09364211b3cfa8991c6e6a4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629204983 2024-12-08T03:40:07,927 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 275f6b3204054b8c88b04a88a1bf316d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629204983 2024-12-08T03:40:07,927 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfaef9917ee043be867defba7fcff5d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629205668 2024-12-08T03:40:07,927 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting e00cce4fec74426392694cd8338db48b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629205668 2024-12-08T03:40:07,928 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e26460095e841bd894fc67b73c2697a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629206061 2024-12-08T03:40:07,928 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59f91c85724c43c99a8fe25e383ad0ef, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629206061 2024-12-08T03:40:07,930 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:07,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:40:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:40:07,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:40:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:07,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/aad23435fcba4fc29a713b416905f97d is 50, key is test_row_0/A:col10/1733629207264/Put/seqid=0 2024-12-08T03:40:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:07,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:07,975 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:07,976 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/d975f4d1333e419898df26401eca3bb9 is 50, key is test_row_0/B:col10/1733629206061/Put/seqid=0 2024-12-08T03:40:07,979 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:07,980 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/c0677cb8722f4a978118d7f861b001f9 is 50, key is test_row_0/A:col10/1733629206061/Put/seqid=0 2024-12-08T03:40:08,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741882_1058 (size=16931) 2024-12-08T03:40:08,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/aad23435fcba4fc29a713b416905f97d 2024-12-08T03:40:08,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741884_1060 (size=12663) 2024-12-08T03:40:08,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1b976979cc094edfbbf6ee90efc4c56c is 50, key is test_row_0/B:col10/1733629207264/Put/seqid=0 2024-12-08T03:40:08,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629268054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629268068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629268071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741883_1059 (size=12663) 2024-12-08T03:40:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741885_1061 (size=12151) 2024-12-08T03:40:08,099 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/d975f4d1333e419898df26401eca3bb9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/d975f4d1333e419898df26401eca3bb9 2024-12-08T03:40:08,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1b976979cc094edfbbf6ee90efc4c56c 2024-12-08T03:40:08,114 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into d975f4d1333e419898df26401eca3bb9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:08,114 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:08,114 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629207917; duration=0sec 2024-12-08T03:40:08,114 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:08,114 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:08,114 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:08,116 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:08,116 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:08,116 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,116 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/87b897606acc4bf6a8a34310c561a3bd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c67c689bfa714e5f9e6ea42d462e0b43, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6de319a9a16c4852b5d6d0e500d993be, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1205c9ce3704b738b8b1e102df73359] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=47.8 K 2024-12-08T03:40:08,117 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b897606acc4bf6a8a34310c561a3bd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733629203742 2024-12-08T03:40:08,117 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c67c689bfa714e5f9e6ea42d462e0b43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629204983 2024-12-08T03:40:08,118 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 6de319a9a16c4852b5d6d0e500d993be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629205668 2024-12-08T03:40:08,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a2c87eca51c949a98115747b4e58ee63 is 50, key is test_row_0/C:col10/1733629207264/Put/seqid=0 2024-12-08T03:40:08,118 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a1205c9ce3704b738b8b1e102df73359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629206061 2024-12-08T03:40:08,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:40:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:08,161 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:08,162 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c08506cbeb064bc4aa1c133d4e106206 is 50, key is test_row_0/C:col10/1733629206061/Put/seqid=0 2024-12-08T03:40:08,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741886_1062 (size=12151) 2024-12-08T03:40:08,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a2c87eca51c949a98115747b4e58ee63 2024-12-08T03:40:08,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629268184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629268189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629268189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/aad23435fcba4fc29a713b416905f97d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/aad23435fcba4fc29a713b416905f97d 2024-12-08T03:40:08,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/aad23435fcba4fc29a713b416905f97d, entries=250, sequenceid=248, filesize=16.5 K 2024-12-08T03:40:08,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1b976979cc094edfbbf6ee90efc4c56c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1b976979cc094edfbbf6ee90efc4c56c 2024-12-08T03:40:08,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741887_1063 (size=12663) 2024-12-08T03:40:08,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1b976979cc094edfbbf6ee90efc4c56c, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T03:40:08,248 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c08506cbeb064bc4aa1c133d4e106206 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c08506cbeb064bc4aa1c133d4e106206 2024-12-08T03:40:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a2c87eca51c949a98115747b4e58ee63 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a2c87eca51c949a98115747b4e58ee63 2024-12-08T03:40:08,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a2c87eca51c949a98115747b4e58ee63, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T03:40:08,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9b6311f5dfb022964ba60e0ab8e515b0 in 328ms, sequenceid=248, compaction requested=false 2024-12-08T03:40:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:08,258 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into c08506cbeb064bc4aa1c133d4e106206(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:08,258 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:08,258 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629207917; duration=0sec 2024-12-08T03:40:08,258 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:08,258 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:40:08,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:40:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,281 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:40:08,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/32b6159c26c640438373bb17c369b8d1 is 50, key is test_row_0/A:col10/1733629208067/Put/seqid=0 2024-12-08T03:40:08,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741888_1064 (size=12301) 2024-12-08T03:40:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/32b6159c26c640438373bb17c369b8d1 2024-12-08T03:40:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/59442dfaca5f4990977643877c6e5f9d is 50, key is test_row_0/B:col10/1733629208067/Put/seqid=0 2024-12-08T03:40:08,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741889_1065 (size=12301) 2024-12-08T03:40:08,400 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/59442dfaca5f4990977643877c6e5f9d 2024-12-08T03:40:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:08,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/392a191879954de1a0beb4ce4685e4db is 50, key is test_row_0/C:col10/1733629208067/Put/seqid=0 2024-12-08T03:40:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629268449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629268449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629268451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,464 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/c0677cb8722f4a978118d7f861b001f9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c0677cb8722f4a978118d7f861b001f9 2024-12-08T03:40:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741890_1066 (size=12301) 2024-12-08T03:40:08,473 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/392a191879954de1a0beb4ce4685e4db 2024-12-08T03:40:08,475 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into c0677cb8722f4a978118d7f861b001f9(size=12.4 K), total size for store is 28.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:08,475 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:08,475 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629207916; duration=0sec 2024-12-08T03:40:08,475 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:08,475 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/32b6159c26c640438373bb17c369b8d1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/32b6159c26c640438373bb17c369b8d1 2024-12-08T03:40:08,486 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/32b6159c26c640438373bb17c369b8d1, entries=150, sequenceid=273, filesize=12.0 K 2024-12-08T03:40:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/59442dfaca5f4990977643877c6e5f9d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/59442dfaca5f4990977643877c6e5f9d 2024-12-08T03:40:08,498 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/59442dfaca5f4990977643877c6e5f9d, entries=150, sequenceid=273, filesize=12.0 K 2024-12-08T03:40:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/392a191879954de1a0beb4ce4685e4db as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/392a191879954de1a0beb4ce4685e4db 2024-12-08T03:40:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/392a191879954de1a0beb4ce4685e4db, entries=150, sequenceid=273, filesize=12.0 K 2024-12-08T03:40:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9b6311f5dfb022964ba60e0ab8e515b0 in 246ms, sequenceid=273, compaction requested=true 2024-12-08T03:40:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-08T03:40:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-08T03:40:08,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T03:40:08,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 872 msec 2024-12-08T03:40:08,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 886 msec 2024-12-08T03:40:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:08,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-08T03:40:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:08,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/706eaeca0991485482692238fac84200 is 50, key is test_row_0/A:col10/1733629208540/Put/seqid=0 2024-12-08T03:40:08,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741891_1067 (size=14741) 2024-12-08T03:40:08,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/706eaeca0991485482692238fac84200 2024-12-08T03:40:08,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/f94c241013e747ce9159d45a89d856c8 is 50, key is test_row_0/B:col10/1733629208540/Put/seqid=0 2024-12-08T03:40:08,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629268612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629268612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629268616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629268624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629268621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741892_1068 (size=12301) 2024-12-08T03:40:08,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/f94c241013e747ce9159d45a89d856c8 2024-12-08T03:40:08,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/b18be2cc67334192ba938b2aa5054636 is 50, key is test_row_0/C:col10/1733629208540/Put/seqid=0 2024-12-08T03:40:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741893_1069 (size=12301) 2024-12-08T03:40:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629268729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629268739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629268741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629268741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629268742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:40:08,789 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:08,789 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-08T03:40:08,792 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:08,793 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:08,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:08,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629268939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:08,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:08,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:08,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:08,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629268948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629268948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629268948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629268958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/b18be2cc67334192ba938b2aa5054636 2024-12-08T03:40:09,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/706eaeca0991485482692238fac84200 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/706eaeca0991485482692238fac84200 2024-12-08T03:40:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:09,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/706eaeca0991485482692238fac84200, entries=200, sequenceid=288, filesize=14.4 K 2024-12-08T03:40:09,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/f94c241013e747ce9159d45a89d856c8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f94c241013e747ce9159d45a89d856c8 2024-12-08T03:40:09,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f94c241013e747ce9159d45a89d856c8, entries=150, sequenceid=288, filesize=12.0 K 2024-12-08T03:40:09,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/b18be2cc67334192ba938b2aa5054636 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b18be2cc67334192ba938b2aa5054636 2024-12-08T03:40:09,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b18be2cc67334192ba938b2aa5054636, entries=150, sequenceid=288, filesize=12.0 K 2024-12-08T03:40:09,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9b6311f5dfb022964ba60e0ab8e515b0 in 583ms, sequenceid=288, compaction requested=true 2024-12-08T03:40:09,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:09,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:09,126 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:09,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:09,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:09,127 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:09,128 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:09,128 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56636 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:09,128 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:09,128 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:09,128 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,128 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,128 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/d975f4d1333e419898df26401eca3bb9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1b976979cc094edfbbf6ee90efc4c56c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/59442dfaca5f4990977643877c6e5f9d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f94c241013e747ce9159d45a89d856c8] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=48.3 K 2024-12-08T03:40:09,128 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c0677cb8722f4a978118d7f861b001f9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/aad23435fcba4fc29a713b416905f97d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/32b6159c26c640438373bb17c369b8d1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/706eaeca0991485482692238fac84200] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=55.3 K 2024-12-08T03:40:09,128 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d975f4d1333e419898df26401eca3bb9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629206061 2024-12-08T03:40:09,129 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0677cb8722f4a978118d7f861b001f9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629206061 2024-12-08T03:40:09,129 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b976979cc094edfbbf6ee90efc4c56c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733629207264 2024-12-08T03:40:09,129 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 59442dfaca5f4990977643877c6e5f9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733629208053 2024-12-08T03:40:09,129 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad23435fcba4fc29a713b416905f97d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733629207259 2024-12-08T03:40:09,130 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f94c241013e747ce9159d45a89d856c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733629208418 2024-12-08T03:40:09,130 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b6159c26c640438373bb17c369b8d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733629208053 2024-12-08T03:40:09,130 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706eaeca0991485482692238fac84200, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733629208418 2024-12-08T03:40:09,140 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:09,141 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/bb437ca391754d05b67c83451a3b68f7 is 50, key is test_row_0/B:col10/1733629208540/Put/seqid=0 2024-12-08T03:40:09,144 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:09,145 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/0978049ca53b4799bb58c2a7230b6fe6 is 50, key is test_row_0/A:col10/1733629208540/Put/seqid=0 2024-12-08T03:40:09,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741895_1071 (size=12949) 2024-12-08T03:40:09,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741894_1070 (size=12949) 2024-12-08T03:40:09,167 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/bb437ca391754d05b67c83451a3b68f7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/bb437ca391754d05b67c83451a3b68f7 2024-12-08T03:40:09,178 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into bb437ca391754d05b67c83451a3b68f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:09,178 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:09,178 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629209126; duration=0sec 2024-12-08T03:40:09,178 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:09,179 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:09,179 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:09,180 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:09,180 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:09,180 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,181 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c08506cbeb064bc4aa1c133d4e106206, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a2c87eca51c949a98115747b4e58ee63, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/392a191879954de1a0beb4ce4685e4db, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b18be2cc67334192ba938b2aa5054636] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=48.3 K 2024-12-08T03:40:09,181 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c08506cbeb064bc4aa1c133d4e106206, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629206061 2024-12-08T03:40:09,181 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c87eca51c949a98115747b4e58ee63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733629207264 2024-12-08T03:40:09,182 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 392a191879954de1a0beb4ce4685e4db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733629208053 2024-12-08T03:40:09,182 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting b18be2cc67334192ba938b2aa5054636, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733629208418 2024-12-08T03:40:09,192 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:09,193 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/519644dfa6644eb487c5246a80af499b is 50, key is test_row_0/C:col10/1733629208540/Put/seqid=0 2024-12-08T03:40:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741896_1072 (size=12949) 2024-12-08T03:40:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:09,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:09,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:40:09,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/669cf55d9a244897a58fb86cbe00ee06 is 50, key is test_row_0/A:col10/1733629208620/Put/seqid=0 2024-12-08T03:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741897_1073 (size=14741) 2024-12-08T03:40:09,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/669cf55d9a244897a58fb86cbe00ee06 2024-12-08T03:40:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629269273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629269272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629269277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/9a58957e85244ff09735d61809d835de is 50, key is test_row_0/B:col10/1733629208620/Put/seqid=0 2024-12-08T03:40:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629269278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629269278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741898_1074 (size=12301) 2024-12-08T03:40:09,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629269388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629269389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629269389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629269389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629269389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:09,561 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/0978049ca53b4799bb58c2a7230b6fe6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/0978049ca53b4799bb58c2a7230b6fe6 2024-12-08T03:40:09,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,575 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 0978049ca53b4799bb58c2a7230b6fe6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:09,575 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:09,575 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629209126; duration=0sec 2024-12-08T03:40:09,575 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:09,575 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:09,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629269598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629269598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629269598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629269598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629269599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,610 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/519644dfa6644eb487c5246a80af499b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/519644dfa6644eb487c5246a80af499b 2024-12-08T03:40:09,618 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into 519644dfa6644eb487c5246a80af499b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:09,618 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:09,618 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629209127; duration=0sec 2024-12-08T03:40:09,618 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:09,618 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:09,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/9a58957e85244ff09735d61809d835de 2024-12-08T03:40:09,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a1f1dbe663f348dbbb985269915e8ed6 is 50, key is test_row_0/C:col10/1733629208620/Put/seqid=0 2024-12-08T03:40:09,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:09,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741899_1075 (size=12301) 2024-12-08T03:40:09,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:09,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:09,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:09,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:09,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629269908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629269908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629269908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629269908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629269908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:10,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:10,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:10,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:10,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:10,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:10,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:10,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a1f1dbe663f348dbbb985269915e8ed6 2024-12-08T03:40:10,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/669cf55d9a244897a58fb86cbe00ee06 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/669cf55d9a244897a58fb86cbe00ee06 2024-12-08T03:40:10,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/669cf55d9a244897a58fb86cbe00ee06, entries=200, sequenceid=312, filesize=14.4 K 2024-12-08T03:40:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/9a58957e85244ff09735d61809d835de as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a58957e85244ff09735d61809d835de 2024-12-08T03:40:10,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a58957e85244ff09735d61809d835de, entries=150, sequenceid=312, filesize=12.0 K 2024-12-08T03:40:10,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/a1f1dbe663f348dbbb985269915e8ed6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1f1dbe663f348dbbb985269915e8ed6 2024-12-08T03:40:10,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1f1dbe663f348dbbb985269915e8ed6, entries=150, sequenceid=312, filesize=12.0 K 2024-12-08T03:40:10,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 9b6311f5dfb022964ba60e0ab8e515b0 in 906ms, sequenceid=312, compaction requested=false 2024-12-08T03:40:10,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:10,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:40:10,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:10,183 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-08T03:40:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/63e811d9b5a240f3bf6f6ca439d2c186 is 50, key is test_row_0/A:col10/1733629209276/Put/seqid=0 2024-12-08T03:40:10,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741900_1076 (size=12301) 2024-12-08T03:40:10,198 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/63e811d9b5a240f3bf6f6ca439d2c186 2024-12-08T03:40:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/894bec053ad648c8952c283d3a2104ba is 50, key is test_row_0/B:col10/1733629209276/Put/seqid=0 2024-12-08T03:40:10,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741901_1077 (size=12301) 2024-12-08T03:40:10,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:10,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629270436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629270443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629270447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629270447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629270447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629270548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629270558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629270559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629270559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629270559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,613 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/894bec053ad648c8952c283d3a2104ba 2024-12-08T03:40:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6e2029cbbe7d417dbe64d0b27920f174 is 50, key is test_row_0/C:col10/1733629209276/Put/seqid=0 2024-12-08T03:40:10,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741902_1078 (size=12301) 2024-12-08T03:40:10,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629270758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629270768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629270768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629270768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629270768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:11,029 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6e2029cbbe7d417dbe64d0b27920f174 2024-12-08T03:40:11,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/63e811d9b5a240f3bf6f6ca439d2c186 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/63e811d9b5a240f3bf6f6ca439d2c186 2024-12-08T03:40:11,042 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/63e811d9b5a240f3bf6f6ca439d2c186, entries=150, sequenceid=328, filesize=12.0 K 2024-12-08T03:40:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/894bec053ad648c8952c283d3a2104ba as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/894bec053ad648c8952c283d3a2104ba 2024-12-08T03:40:11,050 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/894bec053ad648c8952c283d3a2104ba, entries=150, sequenceid=328, filesize=12.0 K 2024-12-08T03:40:11,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6e2029cbbe7d417dbe64d0b27920f174 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6e2029cbbe7d417dbe64d0b27920f174 2024-12-08T03:40:11,058 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6e2029cbbe7d417dbe64d0b27920f174, entries=150, sequenceid=328, filesize=12.0 K 2024-12-08T03:40:11,060 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 9b6311f5dfb022964ba60e0ab8e515b0 in 876ms, sequenceid=328, compaction requested=true 2024-12-08T03:40:11,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:11,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-08T03:40:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-08T03:40:11,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-08T03:40:11,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2680 sec 2024-12-08T03:40:11,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2780 sec 2024-12-08T03:40:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:11,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:11,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:40:11,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/450c8e30d6594e77b9a3179b5a2f8c3b is 50, key is test_row_0/A:col10/1733629210441/Put/seqid=0 2024-12-08T03:40:11,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741903_1079 (size=14741) 2024-12-08T03:40:11,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/450c8e30d6594e77b9a3179b5a2f8c3b 2024-12-08T03:40:11,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629271096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/24fbad0bcff34e60933845844552b6ba is 50, key is test_row_0/B:col10/1733629210441/Put/seqid=0 2024-12-08T03:40:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629271097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629271099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629271100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629271100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741904_1080 (size=12301) 2024-12-08T03:40:11,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/24fbad0bcff34e60933845844552b6ba 2024-12-08T03:40:11,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/e7f48315967f48b8b92d1c87cc1dcaa2 is 50, key is test_row_0/C:col10/1733629210441/Put/seqid=0 2024-12-08T03:40:11,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741905_1081 (size=12301) 2024-12-08T03:40:11,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/e7f48315967f48b8b92d1c87cc1dcaa2 2024-12-08T03:40:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/450c8e30d6594e77b9a3179b5a2f8c3b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/450c8e30d6594e77b9a3179b5a2f8c3b 2024-12-08T03:40:11,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/450c8e30d6594e77b9a3179b5a2f8c3b, entries=200, sequenceid=352, filesize=14.4 K 2024-12-08T03:40:11,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/24fbad0bcff34e60933845844552b6ba as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/24fbad0bcff34e60933845844552b6ba 2024-12-08T03:40:11,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/24fbad0bcff34e60933845844552b6ba, entries=150, sequenceid=352, filesize=12.0 K 2024-12-08T03:40:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/e7f48315967f48b8b92d1c87cc1dcaa2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e7f48315967f48b8b92d1c87cc1dcaa2 2024-12-08T03:40:11,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e7f48315967f48b8b92d1c87cc1dcaa2, entries=150, sequenceid=352, filesize=12.0 K 2024-12-08T03:40:11,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 9b6311f5dfb022964ba60e0ab8e515b0 in 97ms, sequenceid=352, compaction requested=true 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:11,172 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:11,172 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:11,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:11,174 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:11,174 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:11,174 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:11,174 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:11,174 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:11,174 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:11,174 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/bb437ca391754d05b67c83451a3b68f7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a58957e85244ff09735d61809d835de, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/894bec053ad648c8952c283d3a2104ba, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/24fbad0bcff34e60933845844552b6ba] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=48.7 K 2024-12-08T03:40:11,174 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/0978049ca53b4799bb58c2a7230b6fe6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/669cf55d9a244897a58fb86cbe00ee06, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/63e811d9b5a240f3bf6f6ca439d2c186, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/450c8e30d6594e77b9a3179b5a2f8c3b] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=53.4 K 2024-12-08T03:40:11,175 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0978049ca53b4799bb58c2a7230b6fe6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733629208418 2024-12-08T03:40:11,175 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bb437ca391754d05b67c83451a3b68f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733629208418 2024-12-08T03:40:11,175 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 669cf55d9a244897a58fb86cbe00ee06, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733629208611 2024-12-08T03:40:11,175 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a58957e85244ff09735d61809d835de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733629208615 2024-12-08T03:40:11,175 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e811d9b5a240f3bf6f6ca439d2c186, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733629209267 2024-12-08T03:40:11,175 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 894bec053ad648c8952c283d3a2104ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733629209267 2024-12-08T03:40:11,176 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450c8e30d6594e77b9a3179b5a2f8c3b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733629210439 2024-12-08T03:40:11,176 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 24fbad0bcff34e60933845844552b6ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733629210439 2024-12-08T03:40:11,195 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:11,195 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/579e88b0df6346bfa3bae2ad6de00f39 is 50, key is test_row_0/B:col10/1733629210441/Put/seqid=0 2024-12-08T03:40:11,195 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:11,196 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/b9fe45246bd946679d96f592fed93b99 is 50, key is test_row_0/A:col10/1733629210441/Put/seqid=0 2024-12-08T03:40:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741907_1083 (size=13085) 2024-12-08T03:40:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741906_1082 (size=13085) 2024-12-08T03:40:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:11,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:11,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-08T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/476a866420d64ce18cdb49483f296879 is 50, key is test_row_0/A:col10/1733629211094/Put/seqid=0 2024-12-08T03:40:11,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741908_1084 (size=14741) 2024-12-08T03:40:11,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629271231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629271231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629271232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629271233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629271234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629271348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629271348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629271349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629271349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629271349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629271558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629271559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629271559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629271559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629271568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,616 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/b9fe45246bd946679d96f592fed93b99 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b9fe45246bd946679d96f592fed93b99 2024-12-08T03:40:11,617 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/579e88b0df6346bfa3bae2ad6de00f39 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/579e88b0df6346bfa3bae2ad6de00f39 2024-12-08T03:40:11,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/476a866420d64ce18cdb49483f296879 2024-12-08T03:40:11,623 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into b9fe45246bd946679d96f592fed93b99(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:11,623 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,623 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629211172; duration=0sec 2024-12-08T03:40:11,624 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:11,624 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:11,624 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:11,625 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:11,625 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:11,625 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:11,625 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/519644dfa6644eb487c5246a80af499b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1f1dbe663f348dbbb985269915e8ed6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6e2029cbbe7d417dbe64d0b27920f174, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e7f48315967f48b8b92d1c87cc1dcaa2] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=48.7 K 2024-12-08T03:40:11,625 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into 579e88b0df6346bfa3bae2ad6de00f39(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:11,625 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,626 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629211172; duration=0sec 2024-12-08T03:40:11,626 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:11,626 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:11,626 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 519644dfa6644eb487c5246a80af499b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733629208418 2024-12-08T03:40:11,626 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f1dbe663f348dbbb985269915e8ed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733629208615 2024-12-08T03:40:11,626 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2029cbbe7d417dbe64d0b27920f174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733629209267 2024-12-08T03:40:11,627 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f48315967f48b8b92d1c87cc1dcaa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733629210439 2024-12-08T03:40:11,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/f2e3abe191cc47d5ab737f18952d9bbd is 50, key is test_row_0/B:col10/1733629211094/Put/seqid=0 2024-12-08T03:40:11,637 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:11,638 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1e5ec414a12841bcb1d214c8ea677254 is 50, key is test_row_0/C:col10/1733629210441/Put/seqid=0 2024-12-08T03:40:11,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741910_1086 (size=12301) 2024-12-08T03:40:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741909_1085 (size=13085) 2024-12-08T03:40:11,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/f2e3abe191cc47d5ab737f18952d9bbd 2024-12-08T03:40:11,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/b40c19aeae9242c6b19c9bd71b4a89b6 is 50, key is test_row_0/C:col10/1733629211094/Put/seqid=0 2024-12-08T03:40:11,668 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/1e5ec414a12841bcb1d214c8ea677254 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1e5ec414a12841bcb1d214c8ea677254 2024-12-08T03:40:11,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741911_1087 (size=12301) 2024-12-08T03:40:11,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/b40c19aeae9242c6b19c9bd71b4a89b6 2024-12-08T03:40:11,677 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into 1e5ec414a12841bcb1d214c8ea677254(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:11,677 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,677 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629211172; duration=0sec 2024-12-08T03:40:11,678 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:11,678 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:11,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/476a866420d64ce18cdb49483f296879 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/476a866420d64ce18cdb49483f296879 2024-12-08T03:40:11,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/476a866420d64ce18cdb49483f296879, entries=200, sequenceid=367, filesize=14.4 K 2024-12-08T03:40:11,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/f2e3abe191cc47d5ab737f18952d9bbd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f2e3abe191cc47d5ab737f18952d9bbd 2024-12-08T03:40:11,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f2e3abe191cc47d5ab737f18952d9bbd, entries=150, sequenceid=367, filesize=12.0 K 2024-12-08T03:40:11,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/b40c19aeae9242c6b19c9bd71b4a89b6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b40c19aeae9242c6b19c9bd71b4a89b6 2024-12-08T03:40:11,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b40c19aeae9242c6b19c9bd71b4a89b6, entries=150, sequenceid=367, filesize=12.0 K 2024-12-08T03:40:11,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 9b6311f5dfb022964ba60e0ab8e515b0 in 507ms, sequenceid=367, compaction requested=false 2024-12-08T03:40:11,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:11,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:11,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:40:11,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/243e11f31a40402bb08e6c24402e075e is 50, key is test_row_0/A:col10/1733629211233/Put/seqid=0 2024-12-08T03:40:11,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629271884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629271885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741912_1088 (size=12301) 2024-12-08T03:40:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/243e11f31a40402bb08e6c24402e075e 2024-12-08T03:40:11,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629271888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629271894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629271900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:11,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/ae5c83180024440b85fe678097933777 is 50, key is test_row_0/B:col10/1733629211233/Put/seqid=0 2024-12-08T03:40:11,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741913_1089 (size=12301) 2024-12-08T03:40:11,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/ae5c83180024440b85fe678097933777 2024-12-08T03:40:11,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/7273ee1a5e9447d5b9c4df3c28230d96 is 50, key is test_row_0/C:col10/1733629211233/Put/seqid=0 2024-12-08T03:40:11,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741914_1090 (size=12301) 2024-12-08T03:40:11,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/7273ee1a5e9447d5b9c4df3c28230d96 2024-12-08T03:40:11,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/243e11f31a40402bb08e6c24402e075e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/243e11f31a40402bb08e6c24402e075e 2024-12-08T03:40:11,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/243e11f31a40402bb08e6c24402e075e, entries=150, sequenceid=393, filesize=12.0 K 2024-12-08T03:40:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/ae5c83180024440b85fe678097933777 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ae5c83180024440b85fe678097933777 2024-12-08T03:40:11,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ae5c83180024440b85fe678097933777, entries=150, sequenceid=393, filesize=12.0 K 2024-12-08T03:40:11,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/7273ee1a5e9447d5b9c4df3c28230d96 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7273ee1a5e9447d5b9c4df3c28230d96 2024-12-08T03:40:11,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7273ee1a5e9447d5b9c4df3c28230d96, entries=150, sequenceid=393, filesize=12.0 K 2024-12-08T03:40:11,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 9b6311f5dfb022964ba60e0ab8e515b0 in 107ms, sequenceid=393, compaction requested=true 2024-12-08T03:40:11,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:11,979 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:11,980 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:11,981 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:11,981 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:11,981 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:11,981 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:11,981 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:11,981 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:11,981 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b9fe45246bd946679d96f592fed93b99, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/476a866420d64ce18cdb49483f296879, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/243e11f31a40402bb08e6c24402e075e] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=39.2 K 2024-12-08T03:40:11,981 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/579e88b0df6346bfa3bae2ad6de00f39, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f2e3abe191cc47d5ab737f18952d9bbd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ae5c83180024440b85fe678097933777] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=36.8 K 2024-12-08T03:40:11,982 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579e88b0df6346bfa3bae2ad6de00f39, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733629210439 2024-12-08T03:40:11,982 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fe45246bd946679d96f592fed93b99, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733629210439 2024-12-08T03:40:11,982 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2e3abe191cc47d5ab737f18952d9bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733629211094 2024-12-08T03:40:11,983 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 476a866420d64ce18cdb49483f296879, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733629211094 2024-12-08T03:40:11,983 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5c83180024440b85fe678097933777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733629211232 2024-12-08T03:40:11,984 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 243e11f31a40402bb08e6c24402e075e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733629211232 2024-12-08T03:40:11,996 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:11,997 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/7c5b8d4cce1c489fb1c3039be27c01f0 is 50, key is test_row_0/B:col10/1733629211233/Put/seqid=0 2024-12-08T03:40:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:12,000 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:12,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-08T03:40:12,002 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:12,003 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/7374e231e75d4d23ba9f749120fe600c is 50, key is test_row_0/A:col10/1733629211233/Put/seqid=0 2024-12-08T03:40:12,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6ab929367f6a4ed3bc6e4f3efd088ae7 is 50, key is test_row_0/A:col10/1733629211999/Put/seqid=0 2024-12-08T03:40:12,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741916_1092 (size=13187) 2024-12-08T03:40:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741915_1091 (size=13187) 2024-12-08T03:40:12,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629272040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741917_1093 (size=14741) 2024-12-08T03:40:12,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629272044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,055 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/7c5b8d4cce1c489fb1c3039be27c01f0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7c5b8d4cce1c489fb1c3039be27c01f0 2024-12-08T03:40:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629272047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629272055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629272055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,067 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into 7c5b8d4cce1c489fb1c3039be27c01f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:12,068 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:12,068 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=13, startTime=1733629211980; duration=0sec 2024-12-08T03:40:12,068 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:12,068 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:12,068 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:12,070 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:12,070 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:12,070 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:12,070 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1e5ec414a12841bcb1d214c8ea677254, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b40c19aeae9242c6b19c9bd71b4a89b6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7273ee1a5e9447d5b9c4df3c28230d96] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=36.8 K 2024-12-08T03:40:12,071 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5ec414a12841bcb1d214c8ea677254, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733629210439 2024-12-08T03:40:12,074 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40c19aeae9242c6b19c9bd71b4a89b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733629211094 2024-12-08T03:40:12,074 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7273ee1a5e9447d5b9c4df3c28230d96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733629211232 2024-12-08T03:40:12,087 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:12,088 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/dd4b5e616a9d43a6b08133fd6d7fa958 is 50, key is test_row_0/C:col10/1733629211233/Put/seqid=0 2024-12-08T03:40:12,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741918_1094 (size=13187) 2024-12-08T03:40:12,119 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/dd4b5e616a9d43a6b08133fd6d7fa958 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dd4b5e616a9d43a6b08133fd6d7fa958 2024-12-08T03:40:12,131 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into dd4b5e616a9d43a6b08133fd6d7fa958(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:12,131 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:12,131 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=13, startTime=1733629211980; duration=0sec 2024-12-08T03:40:12,132 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:12,132 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629272158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629272159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629272159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629272168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629272168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629272368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629272368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629272368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629272379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629272379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,443 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/7374e231e75d4d23ba9f749120fe600c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/7374e231e75d4d23ba9f749120fe600c 2024-12-08T03:40:12,451 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 7374e231e75d4d23ba9f749120fe600c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:12,451 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:12,451 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=13, startTime=1733629211979; duration=0sec 2024-12-08T03:40:12,452 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:12,452 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:12,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6ab929367f6a4ed3bc6e4f3efd088ae7 2024-12-08T03:40:12,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/c119351c6a7d44b68afb215f7f5c49b7 is 50, key is test_row_0/B:col10/1733629211999/Put/seqid=0 2024-12-08T03:40:12,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741919_1095 (size=12301) 2024-12-08T03:40:12,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/c119351c6a7d44b68afb215f7f5c49b7 2024-12-08T03:40:12,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/4a4b662f71e4402bb86c93cc10713fc1 is 50, key is test_row_0/C:col10/1733629211999/Put/seqid=0 2024-12-08T03:40:12,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741920_1096 (size=12301) 2024-12-08T03:40:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629272678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629272679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629272679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629272688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629272688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:12,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/4a4b662f71e4402bb86c93cc10713fc1 2024-12-08T03:40:12,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6ab929367f6a4ed3bc6e4f3efd088ae7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6ab929367f6a4ed3bc6e4f3efd088ae7 2024-12-08T03:40:12,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6ab929367f6a4ed3bc6e4f3efd088ae7, entries=200, sequenceid=408, filesize=14.4 K 2024-12-08T03:40:12,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/c119351c6a7d44b68afb215f7f5c49b7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/c119351c6a7d44b68afb215f7f5c49b7 2024-12-08T03:40:12,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/c119351c6a7d44b68afb215f7f5c49b7, entries=150, sequenceid=408, filesize=12.0 K 2024-12-08T03:40:12,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/4a4b662f71e4402bb86c93cc10713fc1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4a4b662f71e4402bb86c93cc10713fc1 2024-12-08T03:40:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:40:12,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:12,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-08T03:40:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:40:12,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:12,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:12,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:12,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4a4b662f71e4402bb86c93cc10713fc1, entries=150, sequenceid=408, filesize=12.0 K 2024-12-08T03:40:12,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 9b6311f5dfb022964ba60e0ab8e515b0 in 975ms, sequenceid=408, compaction requested=false 2024-12-08T03:40:12,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:40:13,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:40:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:13,130 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-08T03:40:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/261ec199a7674be8abd2f44b7b8de493 is 50, key is test_row_0/A:col10/1733629212049/Put/seqid=0 2024-12-08T03:40:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741921_1097 (size=12301) 2024-12-08T03:40:13,174 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/261ec199a7674be8abd2f44b7b8de493 2024-12-08T03:40:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:13,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:13,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/08fd6fe713224cffa581f2478c8fd564 is 50, key is test_row_0/B:col10/1733629212049/Put/seqid=0 2024-12-08T03:40:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629273209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629273211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629273211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629273213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629273216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741922_1098 (size=12301) 2024-12-08T03:40:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:40:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629273329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629273329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629273329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629273329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629273329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629273540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629273539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629273541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629273542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629273541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:40:13,653 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/08fd6fe713224cffa581f2478c8fd564 2024-12-08T03:40:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5640529454414f34bd816fb3b2e08ff8 is 50, key is test_row_0/C:col10/1733629212049/Put/seqid=0 2024-12-08T03:40:13,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741923_1099 (size=12301) 2024-12-08T03:40:13,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629273849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629273859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629273859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629273860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629273860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,079 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5640529454414f34bd816fb3b2e08ff8 2024-12-08T03:40:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/261ec199a7674be8abd2f44b7b8de493 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/261ec199a7674be8abd2f44b7b8de493 2024-12-08T03:40:14,105 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/261ec199a7674be8abd2f44b7b8de493, entries=150, sequenceid=433, filesize=12.0 K 2024-12-08T03:40:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/08fd6fe713224cffa581f2478c8fd564 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/08fd6fe713224cffa581f2478c8fd564 2024-12-08T03:40:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:40:14,114 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/08fd6fe713224cffa581f2478c8fd564, entries=150, sequenceid=433, filesize=12.0 K 2024-12-08T03:40:14,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5640529454414f34bd816fb3b2e08ff8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5640529454414f34bd816fb3b2e08ff8 2024-12-08T03:40:14,123 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5640529454414f34bd816fb3b2e08ff8, entries=150, sequenceid=433, filesize=12.0 K 2024-12-08T03:40:14,124 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9b6311f5dfb022964ba60e0ab8e515b0 in 994ms, sequenceid=433, compaction requested=true 2024-12-08T03:40:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-08T03:40:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-08T03:40:14,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T03:40:14,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1600 sec 2024-12-08T03:40:14,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1800 sec 2024-12-08T03:40:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:14,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:14,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-08T03:40:14,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/941e3734d9e9408a856d70fd5305ec47 is 50, key is test_row_0/A:col10/1733629214360/Put/seqid=0 2024-12-08T03:40:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741924_1100 (size=14741) 2024-12-08T03:40:14,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/941e3734d9e9408a856d70fd5305ec47 2024-12-08T03:40:14,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/47d4567176e246dea97ab06711bb8620 is 50, key is test_row_0/B:col10/1733629214360/Put/seqid=0 2024-12-08T03:40:14,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629274403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629274417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629274420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629274421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629274428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741925_1101 (size=12301) 2024-12-08T03:40:14,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/47d4567176e246dea97ab06711bb8620 2024-12-08T03:40:14,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/05fcc4b5a01a4af2860f6cc65ef75d47 is 50, key is test_row_0/C:col10/1733629214360/Put/seqid=0 2024-12-08T03:40:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629274529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741926_1102 (size=12301) 2024-12-08T03:40:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629274538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629274538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629274539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629274550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629274739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629274748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629274748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629274749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629274758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:14,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/05fcc4b5a01a4af2860f6cc65ef75d47 2024-12-08T03:40:14,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/941e3734d9e9408a856d70fd5305ec47 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/941e3734d9e9408a856d70fd5305ec47 2024-12-08T03:40:14,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/941e3734d9e9408a856d70fd5305ec47, entries=200, sequenceid=448, filesize=14.4 K 2024-12-08T03:40:14,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/47d4567176e246dea97ab06711bb8620 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/47d4567176e246dea97ab06711bb8620 2024-12-08T03:40:14,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/47d4567176e246dea97ab06711bb8620, entries=150, sequenceid=448, filesize=12.0 K 2024-12-08T03:40:14,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/05fcc4b5a01a4af2860f6cc65ef75d47 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/05fcc4b5a01a4af2860f6cc65ef75d47 2024-12-08T03:40:14,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/05fcc4b5a01a4af2860f6cc65ef75d47, entries=150, sequenceid=448, filesize=12.0 K 2024-12-08T03:40:14,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 9b6311f5dfb022964ba60e0ab8e515b0 in 632ms, sequenceid=448, compaction requested=true 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:14,997 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:14,997 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:14,998 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:14,998 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:14,998 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:14,998 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:14,999 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:14,999 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:14,999 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7c5b8d4cce1c489fb1c3039be27c01f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/c119351c6a7d44b68afb215f7f5c49b7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/08fd6fe713224cffa581f2478c8fd564, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/47d4567176e246dea97ab06711bb8620] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=48.9 K 2024-12-08T03:40:14,999 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/7374e231e75d4d23ba9f749120fe600c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6ab929367f6a4ed3bc6e4f3efd088ae7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/261ec199a7674be8abd2f44b7b8de493, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/941e3734d9e9408a856d70fd5305ec47] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=53.7 K 2024-12-08T03:40:14,999 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c5b8d4cce1c489fb1c3039be27c01f0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733629211232 2024-12-08T03:40:14,999 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7374e231e75d4d23ba9f749120fe600c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733629211232 2024-12-08T03:40:14,999 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c119351c6a7d44b68afb215f7f5c49b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733629211886 2024-12-08T03:40:14,999 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab929367f6a4ed3bc6e4f3efd088ae7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733629211886 2024-12-08T03:40:15,000 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fd6fe713224cffa581f2478c8fd564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733629212044 2024-12-08T03:40:15,000 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261ec199a7674be8abd2f44b7b8de493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733629212044 2024-12-08T03:40:15,000 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d4567176e246dea97ab06711bb8620, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733629213204 2024-12-08T03:40:15,000 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941e3734d9e9408a856d70fd5305ec47, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733629213204 2024-12-08T03:40:15,025 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:15,025 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/27ab682f114f4457aa0c692cba6800f9 is 50, key is test_row_0/A:col10/1733629214360/Put/seqid=0 2024-12-08T03:40:15,037 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:15,037 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/8e333b356881412e93ed0901f4bc27ad is 50, key is test_row_0/B:col10/1733629214360/Put/seqid=0 2024-12-08T03:40:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:15,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:40:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741928_1104 (size=13323) 2024-12-08T03:40:15,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/a99c8ecd64384b40aff4869267e59608 is 50, key is test_row_0/A:col10/1733629214402/Put/seqid=0 2024-12-08T03:40:15,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741927_1103 (size=13323) 2024-12-08T03:40:15,103 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/27ab682f114f4457aa0c692cba6800f9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/27ab682f114f4457aa0c692cba6800f9 2024-12-08T03:40:15,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741929_1105 (size=12301) 2024-12-08T03:40:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629275078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,117 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 27ab682f114f4457aa0c692cba6800f9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:15,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:15,117 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629214997; duration=0sec 2024-12-08T03:40:15,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:15,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629275080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629275117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:40:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,128 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:15,128 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629275119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,128 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:15,128 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,129 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dd4b5e616a9d43a6b08133fd6d7fa958, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4a4b662f71e4402bb86c93cc10713fc1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5640529454414f34bd816fb3b2e08ff8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/05fcc4b5a01a4af2860f6cc65ef75d47] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=48.9 K 2024-12-08T03:40:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629275119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:15,129 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd4b5e616a9d43a6b08133fd6d7fa958, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733629211232 2024-12-08T03:40:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,130 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4b662f71e4402bb86c93cc10713fc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733629211886 2024-12-08T03:40:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,131 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5640529454414f34bd816fb3b2e08ff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733629212044 2024-12-08T03:40:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-08T03:40:15,133 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05fcc4b5a01a4af2860f6cc65ef75d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733629213204 2024-12-08T03:40:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:40:15,133 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:15,134 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:15,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:15,161 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:15,162 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/4d2b375be0654c1a9af8c0ff3bec9514 is 50, key is test_row_0/C:col10/1733629214360/Put/seqid=0 2024-12-08T03:40:15,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741930_1106 (size=13323) 2024-12-08T03:40:15,220 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/4d2b375be0654c1a9af8c0ff3bec9514 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4d2b375be0654c1a9af8c0ff3bec9514 2024-12-08T03:40:15,231 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into 4d2b375be0654c1a9af8c0ff3bec9514(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:15,232 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:15,232 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629214997; duration=0sec 2024-12-08T03:40:15,232 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:15,232 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:15,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629275229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629275229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:40:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629275239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629275239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629275239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629275438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629275438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:15,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:15,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:40:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629275448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629275448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629275450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,479 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/8e333b356881412e93ed0901f4bc27ad as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8e333b356881412e93ed0901f4bc27ad 2024-12-08T03:40:15,490 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into 8e333b356881412e93ed0901f4bc27ad(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:15,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:15,490 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629214997; duration=0sec 2024-12-08T03:40:15,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:15,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:15,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/a99c8ecd64384b40aff4869267e59608 2024-12-08T03:40:15,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/de676eebe3d54ca3811aad6ef33897a7 is 50, key is test_row_0/B:col10/1733629214402/Put/seqid=0 2024-12-08T03:40:15,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741931_1107 (size=12301) 2024-12-08T03:40:15,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629275748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629275748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:15,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:15,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:40:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629275758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629275758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629275759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:15,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:15,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:15,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/de676eebe3d54ca3811aad6ef33897a7 2024-12-08T03:40:15,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/7f9fc0ad13c3413fa0c597db23ff1aac is 50, key is test_row_0/C:col10/1733629214402/Put/seqid=0 2024-12-08T03:40:16,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741932_1108 (size=12301) 2024-12-08T03:40:16,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629276259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:16,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629276259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:40:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629276268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629276268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629276268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:16,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:16,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:16,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:16,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/7f9fc0ad13c3413fa0c597db23ff1aac 2024-12-08T03:40:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/a99c8ecd64384b40aff4869267e59608 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/a99c8ecd64384b40aff4869267e59608 2024-12-08T03:40:16,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/a99c8ecd64384b40aff4869267e59608, entries=150, sequenceid=471, filesize=12.0 K 2024-12-08T03:40:16,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/de676eebe3d54ca3811aad6ef33897a7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/de676eebe3d54ca3811aad6ef33897a7 2024-12-08T03:40:16,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/de676eebe3d54ca3811aad6ef33897a7, entries=150, sequenceid=471, filesize=12.0 K 2024-12-08T03:40:16,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/7f9fc0ad13c3413fa0c597db23ff1aac as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7f9fc0ad13c3413fa0c597db23ff1aac 2024-12-08T03:40:16,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7f9fc0ad13c3413fa0c597db23ff1aac, entries=150, sequenceid=471, filesize=12.0 K 2024-12-08T03:40:16,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1377ms, sequenceid=471, compaction requested=false 2024-12-08T03:40:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:16,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:40:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:16,527 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-08T03:40:16,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/fc9a4a555d3c47899a0ff95bce55983b is 50, key is test_row_0/A:col10/1733629215084/Put/seqid=0 2024-12-08T03:40:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741933_1109 (size=12301) 2024-12-08T03:40:16,577 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/fc9a4a555d3c47899a0ff95bce55983b 2024-12-08T03:40:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/a10aa75e72274fdb8c6eac62bed93855 is 50, key is test_row_0/B:col10/1733629215084/Put/seqid=0 2024-12-08T03:40:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741934_1110 (size=12301) 2024-12-08T03:40:16,994 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/a10aa75e72274fdb8c6eac62bed93855 2024-12-08T03:40:17,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3b368e97526c4371ac534706896f9278 is 50, key is test_row_0/C:col10/1733629215084/Put/seqid=0 2024-12-08T03:40:17,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741935_1111 (size=12301) 2024-12-08T03:40:17,021 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3b368e97526c4371ac534706896f9278 2024-12-08T03:40:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/fc9a4a555d3c47899a0ff95bce55983b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/fc9a4a555d3c47899a0ff95bce55983b 2024-12-08T03:40:17,036 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/fc9a4a555d3c47899a0ff95bce55983b, entries=150, sequenceid=488, filesize=12.0 K 2024-12-08T03:40:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/a10aa75e72274fdb8c6eac62bed93855 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/a10aa75e72274fdb8c6eac62bed93855 2024-12-08T03:40:17,044 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/a10aa75e72274fdb8c6eac62bed93855, entries=150, sequenceid=488, filesize=12.0 K 2024-12-08T03:40:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3b368e97526c4371ac534706896f9278 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b368e97526c4371ac534706896f9278 2024-12-08T03:40:17,052 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b368e97526c4371ac534706896f9278, entries=150, sequenceid=488, filesize=12.0 K 2024-12-08T03:40:17,053 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 9b6311f5dfb022964ba60e0ab8e515b0 in 526ms, sequenceid=488, compaction requested=true 2024-12-08T03:40:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-08T03:40:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-08T03:40:17,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T03:40:17,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9200 sec 2024-12-08T03:40:17,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9290 sec 2024-12-08T03:40:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:40:17,279 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:17,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-08T03:40:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:40:17,283 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:17,285 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:17,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:17,295 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:17,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:40:17,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6631085f19f241c8a6ef191b2de66dfd is 50, key is test_row_0/A:col10/1733629217284/Put/seqid=0 2024-12-08T03:40:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741936_1112 (size=17181) 2024-12-08T03:40:17,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629277335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629277336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629277337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629277338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629277339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:40:17,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T03:40:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629277450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629277450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629277450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629277450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629277450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:40:17,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T03:40:17,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:17,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629277659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629277659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629277660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629277661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629277668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6631085f19f241c8a6ef191b2de66dfd 2024-12-08T03:40:17,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/9a7d5c54ffdb4e878283b6857eafbb26 is 50, key is test_row_0/B:col10/1733629217284/Put/seqid=0 2024-12-08T03:40:17,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741937_1113 (size=12301) 2024-12-08T03:40:17,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/9a7d5c54ffdb4e878283b6857eafbb26 2024-12-08T03:40:17,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T03:40:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:17,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/e20399da1aa94a33919375eb7050277c is 50, key is test_row_0/C:col10/1733629217284/Put/seqid=0 2024-12-08T03:40:17,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741938_1114 (size=12301) 2024-12-08T03:40:17,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/e20399da1aa94a33919375eb7050277c 2024-12-08T03:40:17,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6631085f19f241c8a6ef191b2de66dfd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6631085f19f241c8a6ef191b2de66dfd 2024-12-08T03:40:17,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6631085f19f241c8a6ef191b2de66dfd, entries=250, sequenceid=500, filesize=16.8 K 2024-12-08T03:40:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/9a7d5c54ffdb4e878283b6857eafbb26 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a7d5c54ffdb4e878283b6857eafbb26 2024-12-08T03:40:17,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a7d5c54ffdb4e878283b6857eafbb26, entries=150, sequenceid=500, filesize=12.0 K 2024-12-08T03:40:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/e20399da1aa94a33919375eb7050277c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e20399da1aa94a33919375eb7050277c 2024-12-08T03:40:17,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e20399da1aa94a33919375eb7050277c, entries=150, sequenceid=500, filesize=12.0 K 2024-12-08T03:40:17,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9b6311f5dfb022964ba60e0ab8e515b0 in 542ms, sequenceid=500, compaction requested=true 2024-12-08T03:40:17,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:17,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:17,837 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:17,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:17,838 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:17,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:17,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:17,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:17,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:17,840 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:17,840 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:17,840 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,840 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8e333b356881412e93ed0901f4bc27ad, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/de676eebe3d54ca3811aad6ef33897a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/a10aa75e72274fdb8c6eac62bed93855, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a7d5c54ffdb4e878283b6857eafbb26] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=49.0 K 2024-12-08T03:40:17,840 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55106 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:17,840 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:17,840 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,841 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/27ab682f114f4457aa0c692cba6800f9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/a99c8ecd64384b40aff4869267e59608, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/fc9a4a555d3c47899a0ff95bce55983b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6631085f19f241c8a6ef191b2de66dfd] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=53.8 K 2024-12-08T03:40:17,841 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e333b356881412e93ed0901f4bc27ad, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733629213204 2024-12-08T03:40:17,841 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27ab682f114f4457aa0c692cba6800f9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733629213204 2024-12-08T03:40:17,841 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting de676eebe3d54ca3811aad6ef33897a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733629214392 2024-12-08T03:40:17,842 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting a99c8ecd64384b40aff4869267e59608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733629214392 2024-12-08T03:40:17,842 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a10aa75e72274fdb8c6eac62bed93855, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1733629215079 2024-12-08T03:40:17,842 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a7d5c54ffdb4e878283b6857eafbb26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733629217284 2024-12-08T03:40:17,842 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc9a4a555d3c47899a0ff95bce55983b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1733629215079 2024-12-08T03:40:17,843 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6631085f19f241c8a6ef191b2de66dfd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733629217280 2024-12-08T03:40:17,873 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:17,874 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/8e9417518e844d359f5079ee9a5e4ab3 is 50, key is test_row_0/A:col10/1733629217284/Put/seqid=0 2024-12-08T03:40:17,880 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:17,880 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/7b2a287c793e45688e88a978c0326ee9 is 50, key is test_row_0/B:col10/1733629217284/Put/seqid=0 2024-12-08T03:40:17,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741939_1115 (size=13459) 2024-12-08T03:40:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:40:17,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T03:40:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,909 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:40:17,915 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/8e9417518e844d359f5079ee9a5e4ab3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8e9417518e844d359f5079ee9a5e4ab3 2024-12-08T03:40:17,930 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 8e9417518e844d359f5079ee9a5e4ab3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:17,930 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:17,930 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=12, startTime=1733629217837; duration=0sec 2024-12-08T03:40:17,931 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:17,931 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:17,931 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:17,933 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:17,933 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:17,933 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:17,934 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4d2b375be0654c1a9af8c0ff3bec9514, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7f9fc0ad13c3413fa0c597db23ff1aac, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b368e97526c4371ac534706896f9278, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e20399da1aa94a33919375eb7050277c] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=49.0 K 2024-12-08T03:40:17,934 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2b375be0654c1a9af8c0ff3bec9514, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733629213204 2024-12-08T03:40:17,935 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f9fc0ad13c3413fa0c597db23ff1aac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733629214392 2024-12-08T03:40:17,936 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b368e97526c4371ac534706896f9278, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1733629215079 2024-12-08T03:40:17,936 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20399da1aa94a33919375eb7050277c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733629217284 2024-12-08T03:40:17,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741940_1116 (size=13459) 2024-12-08T03:40:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/9e7d66df58204811b03ea36cfa8740f1 is 50, key is test_row_0/A:col10/1733629217338/Put/seqid=0 2024-12-08T03:40:17,958 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:17,959 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6dee58ac514a4e38ac7451802c6fcb61 is 50, key is test_row_0/C:col10/1733629217284/Put/seqid=0 2024-12-08T03:40:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:17,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. as already flushing 2024-12-08T03:40:17,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629277981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629277983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629277982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629277985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629277985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741942_1118 (size=12301) 2024-12-08T03:40:18,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741941_1117 (size=13459) 2024-12-08T03:40:18,022 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/6dee58ac514a4e38ac7451802c6fcb61 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6dee58ac514a4e38ac7451802c6fcb61 2024-12-08T03:40:18,032 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into 6dee58ac514a4e38ac7451802c6fcb61(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:18,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:18,032 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=12, startTime=1733629217838; duration=0sec 2024-12-08T03:40:18,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:18,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:18,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629278088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629278098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629278098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629278098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629278108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629278298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629278308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629278308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629278308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629278318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,350 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/7b2a287c793e45688e88a978c0326ee9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7b2a287c793e45688e88a978c0326ee9 2024-12-08T03:40:18,355 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into 7b2a287c793e45688e88a978c0326ee9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:18,355 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:18,355 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=12, startTime=1733629217837; duration=0sec 2024-12-08T03:40:18,355 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:18,356 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:18,393 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/9e7d66df58204811b03ea36cfa8740f1 2024-12-08T03:40:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5860f6701fce41029ea301f9f8b1157f is 50, key is test_row_0/B:col10/1733629217338/Put/seqid=0 2024-12-08T03:40:18,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741943_1119 (size=12301) 2024-12-08T03:40:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:40:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629278608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629278618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629278618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629278618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629278628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:18,815 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5860f6701fce41029ea301f9f8b1157f 2024-12-08T03:40:18,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3b643c6714524b35895acfcfcee02e9c is 50, key is test_row_0/C:col10/1733629217338/Put/seqid=0 2024-12-08T03:40:18,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741944_1120 (size=12301) 2024-12-08T03:40:18,875 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3b643c6714524b35895acfcfcee02e9c 2024-12-08T03:40:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/9e7d66df58204811b03ea36cfa8740f1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9e7d66df58204811b03ea36cfa8740f1 2024-12-08T03:40:18,891 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9e7d66df58204811b03ea36cfa8740f1, entries=150, sequenceid=524, filesize=12.0 K 2024-12-08T03:40:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/5860f6701fce41029ea301f9f8b1157f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5860f6701fce41029ea301f9f8b1157f 2024-12-08T03:40:18,900 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5860f6701fce41029ea301f9f8b1157f, entries=150, sequenceid=524, filesize=12.0 K 2024-12-08T03:40:18,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/3b643c6714524b35895acfcfcee02e9c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b643c6714524b35895acfcfcee02e9c 2024-12-08T03:40:18,914 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b643c6714524b35895acfcfcee02e9c, entries=150, sequenceid=524, filesize=12.0 K 2024-12-08T03:40:18,915 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1006ms, sequenceid=524, compaction requested=false 2024-12-08T03:40:18,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:18,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:18,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-08T03:40:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-08T03:40:18,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-08T03:40:18,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-12-08T03:40:18,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6390 sec 2024-12-08T03:40:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:19,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:40:19,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/ba3ea2105d9b41758fdd16570e50c71e is 50, key is test_row_0/A:col10/1733629217979/Put/seqid=0 2024-12-08T03:40:19,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741945_1121 (size=14741) 2024-12-08T03:40:19,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629279144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629279144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629279146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629279147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629279149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,153 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-08T03:40:19,153 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-08T03:40:19,153 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:19,153 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:19,155 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-08T03:40:19,155 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:19,157 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,157 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,157 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,158 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,158 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:19,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:19,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:19,158 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,160 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-08T03:40:19,160 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:19,160 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,160 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:19,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629279259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629279259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629279259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629279259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629279259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:40:19,430 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629279468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629279468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629279468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629279468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629279469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/ba3ea2105d9b41758fdd16570e50c71e 2024-12-08T03:40:19,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/edafc4c9338246ac968c1981fb875c61 is 50, key is test_row_0/B:col10/1733629217979/Put/seqid=0 2024-12-08T03:40:19,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741946_1122 (size=12301) 2024-12-08T03:40:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629279778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629279778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629279778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629279779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629279778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:19,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/edafc4c9338246ac968c1981fb875c61 2024-12-08T03:40:19,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5c2ab66901fc47d6b8b7c018ec947629 is 50, key is test_row_0/C:col10/1733629217979/Put/seqid=0 2024-12-08T03:40:19,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741947_1123 (size=12301) 2024-12-08T03:40:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54712 deadline: 1733629280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54682 deadline: 1733629280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54702 deadline: 1733629280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54708 deadline: 1733629280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54754 deadline: 1733629280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:20,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5c2ab66901fc47d6b8b7c018ec947629 2024-12-08T03:40:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/ba3ea2105d9b41758fdd16570e50c71e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ba3ea2105d9b41758fdd16570e50c71e 2024-12-08T03:40:20,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ba3ea2105d9b41758fdd16570e50c71e, entries=200, sequenceid=540, filesize=14.4 K 2024-12-08T03:40:20,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/edafc4c9338246ac968c1981fb875c61 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/edafc4c9338246ac968c1981fb875c61 2024-12-08T03:40:20,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/edafc4c9338246ac968c1981fb875c61, entries=150, sequenceid=540, filesize=12.0 K 2024-12-08T03:40:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5c2ab66901fc47d6b8b7c018ec947629 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5c2ab66901fc47d6b8b7c018ec947629 2024-12-08T03:40:20,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5c2ab66901fc47d6b8b7c018ec947629, entries=150, sequenceid=540, filesize=12.0 K 2024-12-08T03:40:20,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1260ms, sequenceid=540, compaction requested=true 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:20,381 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:20,381 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b6311f5dfb022964ba60e0ab8e515b0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:20,382 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:20,382 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:20,382 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/B is initiating minor compaction (all files) 2024-12-08T03:40:20,382 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/A is initiating minor compaction (all files) 2024-12-08T03:40:20,382 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/B in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:20,382 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/A in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:20,382 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7b2a287c793e45688e88a978c0326ee9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5860f6701fce41029ea301f9f8b1157f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/edafc4c9338246ac968c1981fb875c61] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=37.2 K 2024-12-08T03:40:20,382 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8e9417518e844d359f5079ee9a5e4ab3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9e7d66df58204811b03ea36cfa8740f1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ba3ea2105d9b41758fdd16570e50c71e] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=39.6 K 2024-12-08T03:40:20,383 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2a287c793e45688e88a978c0326ee9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733629217284 2024-12-08T03:40:20,383 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9417518e844d359f5079ee9a5e4ab3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733629217284 2024-12-08T03:40:20,383 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5860f6701fce41029ea301f9f8b1157f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1733629217328 2024-12-08T03:40:20,383 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e7d66df58204811b03ea36cfa8740f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1733629217328 2024-12-08T03:40:20,384 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting edafc4c9338246ac968c1981fb875c61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733629217972 2024-12-08T03:40:20,384 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba3ea2105d9b41758fdd16570e50c71e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733629217972 2024-12-08T03:40:20,392 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:20,393 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/3319bec774414b039ccc14ca2b19136e is 50, key is test_row_0/B:col10/1733629217979/Put/seqid=0 2024-12-08T03:40:20,393 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:20,393 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6a6bdd0d687349d4b0922c570a9bc0c1 is 50, key is test_row_0/A:col10/1733629217979/Put/seqid=0 2024-12-08T03:40:20,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741948_1124 (size=13561) 2024-12-08T03:40:20,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741949_1125 (size=13561) 2024-12-08T03:40:20,810 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/3319bec774414b039ccc14ca2b19136e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/3319bec774414b039ccc14ca2b19136e 2024-12-08T03:40:20,810 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/6a6bdd0d687349d4b0922c570a9bc0c1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6a6bdd0d687349d4b0922c570a9bc0c1 2024-12-08T03:40:20,815 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/B of 9b6311f5dfb022964ba60e0ab8e515b0 into 3319bec774414b039ccc14ca2b19136e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:20,815 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/A of 9b6311f5dfb022964ba60e0ab8e515b0 into 6a6bdd0d687349d4b0922c570a9bc0c1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:20,815 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:20,815 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:20,815 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/A, priority=13, startTime=1733629220381; duration=0sec 2024-12-08T03:40:20,815 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/B, priority=13, startTime=1733629220381; duration=0sec 2024-12-08T03:40:20,816 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:20,816 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:20,816 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:A 2024-12-08T03:40:20,816 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:B 2024-12-08T03:40:20,816 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:20,817 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:20,817 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 9b6311f5dfb022964ba60e0ab8e515b0/C is initiating minor compaction (all files) 2024-12-08T03:40:20,817 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b6311f5dfb022964ba60e0ab8e515b0/C in TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:20,817 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6dee58ac514a4e38ac7451802c6fcb61, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b643c6714524b35895acfcfcee02e9c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5c2ab66901fc47d6b8b7c018ec947629] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp, totalSize=37.2 K 2024-12-08T03:40:20,817 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dee58ac514a4e38ac7451802c6fcb61, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733629217284 2024-12-08T03:40:20,818 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b643c6714524b35895acfcfcee02e9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1733629217328 2024-12-08T03:40:20,818 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c2ab66901fc47d6b8b7c018ec947629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733629217972 2024-12-08T03:40:20,825 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b6311f5dfb022964ba60e0ab8e515b0#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:20,825 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5a79f44618df4c038ce4e7bae8bc0f66 is 50, key is test_row_0/C:col10/1733629217979/Put/seqid=0 2024-12-08T03:40:20,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741950_1126 (size=13561) 2024-12-08T03:40:21,242 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/5a79f44618df4c038ce4e7bae8bc0f66 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5a79f44618df4c038ce4e7bae8bc0f66 2024-12-08T03:40:21,249 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b6311f5dfb022964ba60e0ab8e515b0/C of 9b6311f5dfb022964ba60e0ab8e515b0 into 5a79f44618df4c038ce4e7bae8bc0f66(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:21,249 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:21,249 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., storeName=9b6311f5dfb022964ba60e0ab8e515b0/C, priority=13, startTime=1733629220381; duration=0sec 2024-12-08T03:40:21,249 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:21,249 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b6311f5dfb022964ba60e0ab8e515b0:C 2024-12-08T03:40:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:21,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:40:21,301 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-08T03:40:21,301 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-08T03:40:21,301 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-08T03:40:21,301 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:21,301 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:21,301 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:21,301 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,301 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,301 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,301 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,301 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:21,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:21,302 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:21,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/77f2301cc9eb4146a4cf34d701587bbf is 50, key is test_row_0/A:col10/1733629221298/Put/seqid=0 2024-12-08T03:40:21,311 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-08T03:40:21,312 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:21,312 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,312 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:21,312 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-08T03:40:21,312 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:21,312 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,312 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3460 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3386 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1630 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4886 rows 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1619 2024-12-08T03:40:21,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4853 rows 2024-12-08T03:40:21,313 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:40:21,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:40:21,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:21,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:21,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741951_1127 (size=12301) 2024-12-08T03:40:21,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:40:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:21,323 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629221323"}]},"ts":"1733629221323"} 2024-12-08T03:40:21,325 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:40:21,325 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:40:21,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:40:21,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, UNASSIGN}] 2024-12-08T03:40:21,331 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, UNASSIGN 2024-12-08T03:40:21,333 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9b6311f5dfb022964ba60e0ab8e515b0, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:21,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, UNASSIGN because future has completed 2024-12-08T03:40:21,335 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:40:21,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:40:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:21,494 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:21,495 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:40:21,496 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 9b6311f5dfb022964ba60e0ab8e515b0, disabling compactions & flushes 2024-12-08T03:40:21,496 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:21,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/77f2301cc9eb4146a4cf34d701587bbf 2024-12-08T03:40:21,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/acf39626c6844256872440faac60b333 is 50, key is test_row_0/B:col10/1733629221298/Put/seqid=0 2024-12-08T03:40:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741952_1128 (size=12301) 2024-12-08T03:40:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:22,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/acf39626c6844256872440faac60b333 2024-12-08T03:40:22,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c9f3694cec3e48a88805fc2e7d0ee29e is 50, key is test_row_0/C:col10/1733629221298/Put/seqid=0 2024-12-08T03:40:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741953_1129 (size=12301) 2024-12-08T03:40:22,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:40:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:22,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c9f3694cec3e48a88805fc2e7d0ee29e 2024-12-08T03:40:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/77f2301cc9eb4146a4cf34d701587bbf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/77f2301cc9eb4146a4cf34d701587bbf 2024-12-08T03:40:22,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/77f2301cc9eb4146a4cf34d701587bbf, entries=150, sequenceid=569, filesize=12.0 K 2024-12-08T03:40:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/acf39626c6844256872440faac60b333 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/acf39626c6844256872440faac60b333 2024-12-08T03:40:22,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/acf39626c6844256872440faac60b333, entries=150, sequenceid=569, filesize=12.0 K 2024-12-08T03:40:22,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/c9f3694cec3e48a88805fc2e7d0ee29e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c9f3694cec3e48a88805fc2e7d0ee29e 2024-12-08T03:40:22,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c9f3694cec3e48a88805fc2e7d0ee29e, entries=150, sequenceid=569, filesize=12.0 K 2024-12-08T03:40:22,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=13.42 KB/13740 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1279ms, sequenceid=569, compaction requested=false 2024-12-08T03:40:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b6311f5dfb022964ba60e0ab8e515b0: 2024-12-08T03:40:22,579 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:22,579 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:22,579 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. after waiting 0 ms 2024-12-08T03:40:22,579 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:22,579 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 9b6311f5dfb022964ba60e0ab8e515b0 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-12-08T03:40:22,583 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/3f307521552849009896a1f77de97927 is 50, key is test_row_0/A:col10/1733629221309/Put/seqid=0 2024-12-08T03:40:22,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741954_1130 (size=9857) 2024-12-08T03:40:22,988 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/3f307521552849009896a1f77de97927 2024-12-08T03:40:22,995 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1365f65c37a14945a7e1627ddb45cf86 is 50, key is test_row_0/B:col10/1733629221309/Put/seqid=0 2024-12-08T03:40:22,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741955_1131 (size=9857) 2024-12-08T03:40:23,401 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1365f65c37a14945a7e1627ddb45cf86 2024-12-08T03:40:23,416 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/49a20db218f44ee983e6ef6d611be4df is 50, key is test_row_0/C:col10/1733629221309/Put/seqid=0 2024-12-08T03:40:23,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741956_1132 (size=9857) 2024-12-08T03:40:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:23,821 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/49a20db218f44ee983e6ef6d611be4df 2024-12-08T03:40:23,826 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/A/3f307521552849009896a1f77de97927 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/3f307521552849009896a1f77de97927 2024-12-08T03:40:23,831 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/3f307521552849009896a1f77de97927, entries=100, sequenceid=574, filesize=9.6 K 2024-12-08T03:40:23,832 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/B/1365f65c37a14945a7e1627ddb45cf86 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1365f65c37a14945a7e1627ddb45cf86 2024-12-08T03:40:23,837 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1365f65c37a14945a7e1627ddb45cf86, entries=100, sequenceid=574, filesize=9.6 K 2024-12-08T03:40:23,838 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/.tmp/C/49a20db218f44ee983e6ef6d611be4df as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/49a20db218f44ee983e6ef6d611be4df 2024-12-08T03:40:23,841 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/49a20db218f44ee983e6ef6d611be4df, entries=100, sequenceid=574, filesize=9.6 K 2024-12-08T03:40:23,842 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1263ms, sequenceid=574, compaction requested=true 2024-12-08T03:40:23,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/28ebf1387033438e94f0558e571e1781, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/38703505d42340cbbfb5d010c139137e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/585fc22ada0a4dd8aff613d3e5d81f28, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/19ad3467b3e14d1c897af3c5b41801ec, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ccac79cf9cf54bcab7c1e13893a0982a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b75114fa0e9746f5a8ca724b9c26f20c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9b99dd61e69b46d28de753ba8c648858, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8836a8e36d664953afa0bc3048b3dd94, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/82eade7a64b342acbc67e0f73447eea9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c14fdef30ee441b78cbb772d2ff5c58b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/89ddb2561d7e488ba54e8b0fbc137253, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/d8eb39c09efa4beabb5a59a1979e89e6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cbfea11f09364211b3cfa8991c6e6a4c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cfaef9917ee043be867defba7fcff5d5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/59f91c85724c43c99a8fe25e383ad0ef, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c0677cb8722f4a978118d7f861b001f9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/aad23435fcba4fc29a713b416905f97d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/32b6159c26c640438373bb17c369b8d1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/706eaeca0991485482692238fac84200, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/0978049ca53b4799bb58c2a7230b6fe6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/669cf55d9a244897a58fb86cbe00ee06, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/63e811d9b5a240f3bf6f6ca439d2c186, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/450c8e30d6594e77b9a3179b5a2f8c3b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b9fe45246bd946679d96f592fed93b99, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/476a866420d64ce18cdb49483f296879, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/7374e231e75d4d23ba9f749120fe600c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/243e11f31a40402bb08e6c24402e075e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6ab929367f6a4ed3bc6e4f3efd088ae7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/261ec199a7674be8abd2f44b7b8de493, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/941e3734d9e9408a856d70fd5305ec47, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/27ab682f114f4457aa0c692cba6800f9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/a99c8ecd64384b40aff4869267e59608, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/fc9a4a555d3c47899a0ff95bce55983b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6631085f19f241c8a6ef191b2de66dfd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8e9417518e844d359f5079ee9a5e4ab3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9e7d66df58204811b03ea36cfa8740f1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ba3ea2105d9b41758fdd16570e50c71e] to archive 2024-12-08T03:40:23,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:40:23,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/28ebf1387033438e94f0558e571e1781 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/28ebf1387033438e94f0558e571e1781 2024-12-08T03:40:23,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/38703505d42340cbbfb5d010c139137e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/38703505d42340cbbfb5d010c139137e 2024-12-08T03:40:23,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/585fc22ada0a4dd8aff613d3e5d81f28 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/585fc22ada0a4dd8aff613d3e5d81f28 2024-12-08T03:40:23,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/19ad3467b3e14d1c897af3c5b41801ec to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/19ad3467b3e14d1c897af3c5b41801ec 2024-12-08T03:40:23,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ccac79cf9cf54bcab7c1e13893a0982a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ccac79cf9cf54bcab7c1e13893a0982a 2024-12-08T03:40:23,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b75114fa0e9746f5a8ca724b9c26f20c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b75114fa0e9746f5a8ca724b9c26f20c 2024-12-08T03:40:23,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9b99dd61e69b46d28de753ba8c648858 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9b99dd61e69b46d28de753ba8c648858 2024-12-08T03:40:23,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8836a8e36d664953afa0bc3048b3dd94 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8836a8e36d664953afa0bc3048b3dd94 2024-12-08T03:40:23,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/82eade7a64b342acbc67e0f73447eea9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/82eade7a64b342acbc67e0f73447eea9 2024-12-08T03:40:23,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c14fdef30ee441b78cbb772d2ff5c58b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c14fdef30ee441b78cbb772d2ff5c58b 2024-12-08T03:40:23,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/89ddb2561d7e488ba54e8b0fbc137253 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/89ddb2561d7e488ba54e8b0fbc137253 2024-12-08T03:40:23,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/d8eb39c09efa4beabb5a59a1979e89e6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/d8eb39c09efa4beabb5a59a1979e89e6 2024-12-08T03:40:23,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cbfea11f09364211b3cfa8991c6e6a4c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cbfea11f09364211b3cfa8991c6e6a4c 2024-12-08T03:40:23,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cfaef9917ee043be867defba7fcff5d5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/cfaef9917ee043be867defba7fcff5d5 2024-12-08T03:40:23,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/59f91c85724c43c99a8fe25e383ad0ef to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/59f91c85724c43c99a8fe25e383ad0ef 2024-12-08T03:40:23,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c0677cb8722f4a978118d7f861b001f9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/c0677cb8722f4a978118d7f861b001f9 2024-12-08T03:40:23,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/aad23435fcba4fc29a713b416905f97d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/aad23435fcba4fc29a713b416905f97d 2024-12-08T03:40:23,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/32b6159c26c640438373bb17c369b8d1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/32b6159c26c640438373bb17c369b8d1 2024-12-08T03:40:23,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/706eaeca0991485482692238fac84200 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/706eaeca0991485482692238fac84200 2024-12-08T03:40:23,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/0978049ca53b4799bb58c2a7230b6fe6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/0978049ca53b4799bb58c2a7230b6fe6 2024-12-08T03:40:23,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/669cf55d9a244897a58fb86cbe00ee06 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/669cf55d9a244897a58fb86cbe00ee06 2024-12-08T03:40:23,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/63e811d9b5a240f3bf6f6ca439d2c186 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/63e811d9b5a240f3bf6f6ca439d2c186 2024-12-08T03:40:23,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/450c8e30d6594e77b9a3179b5a2f8c3b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/450c8e30d6594e77b9a3179b5a2f8c3b 2024-12-08T03:40:23,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b9fe45246bd946679d96f592fed93b99 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/b9fe45246bd946679d96f592fed93b99 2024-12-08T03:40:23,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/476a866420d64ce18cdb49483f296879 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/476a866420d64ce18cdb49483f296879 2024-12-08T03:40:23,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/7374e231e75d4d23ba9f749120fe600c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/7374e231e75d4d23ba9f749120fe600c 2024-12-08T03:40:23,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/243e11f31a40402bb08e6c24402e075e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/243e11f31a40402bb08e6c24402e075e 2024-12-08T03:40:23,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6ab929367f6a4ed3bc6e4f3efd088ae7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6ab929367f6a4ed3bc6e4f3efd088ae7 2024-12-08T03:40:23,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/261ec199a7674be8abd2f44b7b8de493 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/261ec199a7674be8abd2f44b7b8de493 2024-12-08T03:40:23,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/941e3734d9e9408a856d70fd5305ec47 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/941e3734d9e9408a856d70fd5305ec47 2024-12-08T03:40:23,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/27ab682f114f4457aa0c692cba6800f9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/27ab682f114f4457aa0c692cba6800f9 2024-12-08T03:40:23,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/a99c8ecd64384b40aff4869267e59608 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/a99c8ecd64384b40aff4869267e59608 2024-12-08T03:40:23,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/fc9a4a555d3c47899a0ff95bce55983b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/fc9a4a555d3c47899a0ff95bce55983b 2024-12-08T03:40:23,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6631085f19f241c8a6ef191b2de66dfd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6631085f19f241c8a6ef191b2de66dfd 2024-12-08T03:40:23,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8e9417518e844d359f5079ee9a5e4ab3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/8e9417518e844d359f5079ee9a5e4ab3 2024-12-08T03:40:23,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9e7d66df58204811b03ea36cfa8740f1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/9e7d66df58204811b03ea36cfa8740f1 2024-12-08T03:40:23,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ba3ea2105d9b41758fdd16570e50c71e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/ba3ea2105d9b41758fdd16570e50c71e 2024-12-08T03:40:23,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/aa967572abbb4a6e9b698270ace56df8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/fae4d1dfe5c34787bb59187691fa5a1c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/87c53af07e6b4f6e8c1b46de9b9634ce, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/cbdf676ff6694c63bcd13198a91d50f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/559cc522733f42dc95beda2ec99496be, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/53790a75fe2f4582b9038912a53735b3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/b76d67e07710475a9d4aa2f131313308, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/68877633022a4eb9810761e3f041d0c4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ca5263f92aed4417a2e964b417b408e0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8fc1094994994ccaab521de403e36f2a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5565c5ea548340a98eb95638f251f72b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/275f6b3204054b8c88b04a88a1bf316d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/e00cce4fec74426392694cd8338db48b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/d975f4d1333e419898df26401eca3bb9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1e26460095e841bd894fc67b73c2697a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1b976979cc094edfbbf6ee90efc4c56c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/59442dfaca5f4990977643877c6e5f9d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/bb437ca391754d05b67c83451a3b68f7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f94c241013e747ce9159d45a89d856c8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a58957e85244ff09735d61809d835de, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/894bec053ad648c8952c283d3a2104ba, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/579e88b0df6346bfa3bae2ad6de00f39, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/24fbad0bcff34e60933845844552b6ba, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f2e3abe191cc47d5ab737f18952d9bbd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7c5b8d4cce1c489fb1c3039be27c01f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ae5c83180024440b85fe678097933777, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/c119351c6a7d44b68afb215f7f5c49b7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/08fd6fe713224cffa581f2478c8fd564, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8e333b356881412e93ed0901f4bc27ad, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/47d4567176e246dea97ab06711bb8620, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/de676eebe3d54ca3811aad6ef33897a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/a10aa75e72274fdb8c6eac62bed93855, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7b2a287c793e45688e88a978c0326ee9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a7d5c54ffdb4e878283b6857eafbb26, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5860f6701fce41029ea301f9f8b1157f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/edafc4c9338246ac968c1981fb875c61] to archive 2024-12-08T03:40:23,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:40:23,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/aa967572abbb4a6e9b698270ace56df8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/aa967572abbb4a6e9b698270ace56df8 2024-12-08T03:40:23,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/fae4d1dfe5c34787bb59187691fa5a1c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/fae4d1dfe5c34787bb59187691fa5a1c 2024-12-08T03:40:23,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/87c53af07e6b4f6e8c1b46de9b9634ce to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/87c53af07e6b4f6e8c1b46de9b9634ce 2024-12-08T03:40:23,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/cbdf676ff6694c63bcd13198a91d50f0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/cbdf676ff6694c63bcd13198a91d50f0 2024-12-08T03:40:23,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/559cc522733f42dc95beda2ec99496be to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/559cc522733f42dc95beda2ec99496be 2024-12-08T03:40:23,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/53790a75fe2f4582b9038912a53735b3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/53790a75fe2f4582b9038912a53735b3 2024-12-08T03:40:23,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/b76d67e07710475a9d4aa2f131313308 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/b76d67e07710475a9d4aa2f131313308 2024-12-08T03:40:23,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/68877633022a4eb9810761e3f041d0c4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/68877633022a4eb9810761e3f041d0c4 2024-12-08T03:40:23,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ca5263f92aed4417a2e964b417b408e0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ca5263f92aed4417a2e964b417b408e0 2024-12-08T03:40:23,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5e2bf87e1fdb4f45a5dfa7eb26e9d7e4 2024-12-08T03:40:23,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8fc1094994994ccaab521de403e36f2a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8fc1094994994ccaab521de403e36f2a 2024-12-08T03:40:23,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5565c5ea548340a98eb95638f251f72b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5565c5ea548340a98eb95638f251f72b 2024-12-08T03:40:23,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/275f6b3204054b8c88b04a88a1bf316d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/275f6b3204054b8c88b04a88a1bf316d 2024-12-08T03:40:23,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/e00cce4fec74426392694cd8338db48b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/e00cce4fec74426392694cd8338db48b 2024-12-08T03:40:23,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/d975f4d1333e419898df26401eca3bb9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/d975f4d1333e419898df26401eca3bb9 2024-12-08T03:40:23,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1e26460095e841bd894fc67b73c2697a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1e26460095e841bd894fc67b73c2697a 2024-12-08T03:40:23,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1b976979cc094edfbbf6ee90efc4c56c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1b976979cc094edfbbf6ee90efc4c56c 2024-12-08T03:40:23,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/59442dfaca5f4990977643877c6e5f9d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/59442dfaca5f4990977643877c6e5f9d 2024-12-08T03:40:23,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/bb437ca391754d05b67c83451a3b68f7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/bb437ca391754d05b67c83451a3b68f7 2024-12-08T03:40:23,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f94c241013e747ce9159d45a89d856c8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f94c241013e747ce9159d45a89d856c8 2024-12-08T03:40:23,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a58957e85244ff09735d61809d835de to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a58957e85244ff09735d61809d835de 2024-12-08T03:40:23,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/894bec053ad648c8952c283d3a2104ba to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/894bec053ad648c8952c283d3a2104ba 2024-12-08T03:40:23,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/579e88b0df6346bfa3bae2ad6de00f39 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/579e88b0df6346bfa3bae2ad6de00f39 2024-12-08T03:40:23,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/24fbad0bcff34e60933845844552b6ba to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/24fbad0bcff34e60933845844552b6ba 2024-12-08T03:40:23,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f2e3abe191cc47d5ab737f18952d9bbd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/f2e3abe191cc47d5ab737f18952d9bbd 2024-12-08T03:40:23,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7c5b8d4cce1c489fb1c3039be27c01f0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7c5b8d4cce1c489fb1c3039be27c01f0 2024-12-08T03:40:23,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ae5c83180024440b85fe678097933777 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/ae5c83180024440b85fe678097933777 2024-12-08T03:40:23,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/c119351c6a7d44b68afb215f7f5c49b7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/c119351c6a7d44b68afb215f7f5c49b7 2024-12-08T03:40:23,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/08fd6fe713224cffa581f2478c8fd564 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/08fd6fe713224cffa581f2478c8fd564 2024-12-08T03:40:23,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8e333b356881412e93ed0901f4bc27ad to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/8e333b356881412e93ed0901f4bc27ad 2024-12-08T03:40:23,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/47d4567176e246dea97ab06711bb8620 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/47d4567176e246dea97ab06711bb8620 2024-12-08T03:40:23,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/de676eebe3d54ca3811aad6ef33897a7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/de676eebe3d54ca3811aad6ef33897a7 2024-12-08T03:40:23,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/a10aa75e72274fdb8c6eac62bed93855 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/a10aa75e72274fdb8c6eac62bed93855 2024-12-08T03:40:23,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7b2a287c793e45688e88a978c0326ee9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/7b2a287c793e45688e88a978c0326ee9 2024-12-08T03:40:23,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a7d5c54ffdb4e878283b6857eafbb26 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/9a7d5c54ffdb4e878283b6857eafbb26 2024-12-08T03:40:23,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5860f6701fce41029ea301f9f8b1157f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/5860f6701fce41029ea301f9f8b1157f 2024-12-08T03:40:23,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/edafc4c9338246ac968c1981fb875c61 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/edafc4c9338246ac968c1981fb875c61 2024-12-08T03:40:23,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/623af4aad8e6422eaa490d491169da61, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dde156b9767541b2a5a246b152f0e850, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/8bc43cd3160645b99cab0c324301efc6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f9da3e3f9d504fc8b87861a852280a4b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a13773d32eb24da8ba8e9cd03a681ba1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1eb2d7bd385441c49c1c213b462556ac, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4e8f11b721741e48b72b2556b21cce6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4ba77b57f8d49d79bf07380874b6d2c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1a564ac9d4c24fc38058a3f7a9a9bc1d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/87b897606acc4bf6a8a34310c561a3bd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/894366f13a9c4ab5a210644de5ec3e1c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c67c689bfa714e5f9e6ea42d462e0b43, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6de319a9a16c4852b5d6d0e500d993be, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c08506cbeb064bc4aa1c133d4e106206, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1205c9ce3704b738b8b1e102df73359, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a2c87eca51c949a98115747b4e58ee63, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/392a191879954de1a0beb4ce4685e4db, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/519644dfa6644eb487c5246a80af499b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b18be2cc67334192ba938b2aa5054636, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1f1dbe663f348dbbb985269915e8ed6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6e2029cbbe7d417dbe64d0b27920f174, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1e5ec414a12841bcb1d214c8ea677254, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e7f48315967f48b8b92d1c87cc1dcaa2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b40c19aeae9242c6b19c9bd71b4a89b6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dd4b5e616a9d43a6b08133fd6d7fa958, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7273ee1a5e9447d5b9c4df3c28230d96, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4a4b662f71e4402bb86c93cc10713fc1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5640529454414f34bd816fb3b2e08ff8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4d2b375be0654c1a9af8c0ff3bec9514, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/05fcc4b5a01a4af2860f6cc65ef75d47, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7f9fc0ad13c3413fa0c597db23ff1aac, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b368e97526c4371ac534706896f9278, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6dee58ac514a4e38ac7451802c6fcb61, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e20399da1aa94a33919375eb7050277c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b643c6714524b35895acfcfcee02e9c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5c2ab66901fc47d6b8b7c018ec947629] to archive 2024-12-08T03:40:23,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:40:23,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/623af4aad8e6422eaa490d491169da61 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/623af4aad8e6422eaa490d491169da61 2024-12-08T03:40:23,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dde156b9767541b2a5a246b152f0e850 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dde156b9767541b2a5a246b152f0e850 2024-12-08T03:40:23,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/8bc43cd3160645b99cab0c324301efc6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/8bc43cd3160645b99cab0c324301efc6 2024-12-08T03:40:23,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f9da3e3f9d504fc8b87861a852280a4b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f9da3e3f9d504fc8b87861a852280a4b 2024-12-08T03:40:23,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a13773d32eb24da8ba8e9cd03a681ba1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a13773d32eb24da8ba8e9cd03a681ba1 2024-12-08T03:40:23,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1eb2d7bd385441c49c1c213b462556ac to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1eb2d7bd385441c49c1c213b462556ac 2024-12-08T03:40:23,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4e8f11b721741e48b72b2556b21cce6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4e8f11b721741e48b72b2556b21cce6 2024-12-08T03:40:23,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4ba77b57f8d49d79bf07380874b6d2c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/f4ba77b57f8d49d79bf07380874b6d2c 2024-12-08T03:40:23,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3ca5a3e20d8c4bffb7ccb77f79c5c9f4 2024-12-08T03:40:23,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1a564ac9d4c24fc38058a3f7a9a9bc1d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1a564ac9d4c24fc38058a3f7a9a9bc1d 2024-12-08T03:40:23,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/87b897606acc4bf6a8a34310c561a3bd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/87b897606acc4bf6a8a34310c561a3bd 2024-12-08T03:40:23,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/894366f13a9c4ab5a210644de5ec3e1c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/894366f13a9c4ab5a210644de5ec3e1c 2024-12-08T03:40:23,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c67c689bfa714e5f9e6ea42d462e0b43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c67c689bfa714e5f9e6ea42d462e0b43 2024-12-08T03:40:23,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6de319a9a16c4852b5d6d0e500d993be to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6de319a9a16c4852b5d6d0e500d993be 2024-12-08T03:40:23,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c08506cbeb064bc4aa1c133d4e106206 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c08506cbeb064bc4aa1c133d4e106206 2024-12-08T03:40:23,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1205c9ce3704b738b8b1e102df73359 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1205c9ce3704b738b8b1e102df73359 2024-12-08T03:40:23,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a2c87eca51c949a98115747b4e58ee63 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a2c87eca51c949a98115747b4e58ee63 2024-12-08T03:40:23,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/392a191879954de1a0beb4ce4685e4db to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/392a191879954de1a0beb4ce4685e4db 2024-12-08T03:40:23,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/519644dfa6644eb487c5246a80af499b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/519644dfa6644eb487c5246a80af499b 2024-12-08T03:40:23,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b18be2cc67334192ba938b2aa5054636 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b18be2cc67334192ba938b2aa5054636 2024-12-08T03:40:23,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1f1dbe663f348dbbb985269915e8ed6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/a1f1dbe663f348dbbb985269915e8ed6 2024-12-08T03:40:23,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6e2029cbbe7d417dbe64d0b27920f174 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6e2029cbbe7d417dbe64d0b27920f174 2024-12-08T03:40:23,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1e5ec414a12841bcb1d214c8ea677254 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/1e5ec414a12841bcb1d214c8ea677254 2024-12-08T03:40:23,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e7f48315967f48b8b92d1c87cc1dcaa2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e7f48315967f48b8b92d1c87cc1dcaa2 2024-12-08T03:40:23,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b40c19aeae9242c6b19c9bd71b4a89b6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/b40c19aeae9242c6b19c9bd71b4a89b6 2024-12-08T03:40:23,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dd4b5e616a9d43a6b08133fd6d7fa958 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/dd4b5e616a9d43a6b08133fd6d7fa958 2024-12-08T03:40:23,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7273ee1a5e9447d5b9c4df3c28230d96 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7273ee1a5e9447d5b9c4df3c28230d96 2024-12-08T03:40:23,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4a4b662f71e4402bb86c93cc10713fc1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4a4b662f71e4402bb86c93cc10713fc1 2024-12-08T03:40:23,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5640529454414f34bd816fb3b2e08ff8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5640529454414f34bd816fb3b2e08ff8 2024-12-08T03:40:23,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4d2b375be0654c1a9af8c0ff3bec9514 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/4d2b375be0654c1a9af8c0ff3bec9514 2024-12-08T03:40:23,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/05fcc4b5a01a4af2860f6cc65ef75d47 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/05fcc4b5a01a4af2860f6cc65ef75d47 2024-12-08T03:40:23,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7f9fc0ad13c3413fa0c597db23ff1aac to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/7f9fc0ad13c3413fa0c597db23ff1aac 2024-12-08T03:40:23,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b368e97526c4371ac534706896f9278 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b368e97526c4371ac534706896f9278 2024-12-08T03:40:23,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6dee58ac514a4e38ac7451802c6fcb61 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/6dee58ac514a4e38ac7451802c6fcb61 2024-12-08T03:40:23,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e20399da1aa94a33919375eb7050277c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/e20399da1aa94a33919375eb7050277c 2024-12-08T03:40:23,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b643c6714524b35895acfcfcee02e9c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/3b643c6714524b35895acfcfcee02e9c 2024-12-08T03:40:23,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5c2ab66901fc47d6b8b7c018ec947629 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5c2ab66901fc47d6b8b7c018ec947629 2024-12-08T03:40:23,998 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/recovered.edits/577.seqid, newMaxSeqId=577, maxSeqId=1 2024-12-08T03:40:23,999 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. 2024-12-08T03:40:23,999 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 9b6311f5dfb022964ba60e0ab8e515b0: Waiting for close lock at 1733629221496Running coprocessor pre-close hooks at 1733629221496Disabling compacts and flushes for region at 1733629221496Disabling writes for close at 1733629222579 (+1083 ms)Obtaining lock to block concurrent updates at 1733629222579Preparing flush snapshotting stores in 9b6311f5dfb022964ba60e0ab8e515b0 at 1733629222579Finished memstore snapshotting TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1733629222580 (+1 ms)Flushing stores of TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0. at 1733629222580Flushing 9b6311f5dfb022964ba60e0ab8e515b0/A: creating writer at 1733629222580Flushing 9b6311f5dfb022964ba60e0ab8e515b0/A: appending metadata at 1733629222582 (+2 ms)Flushing 9b6311f5dfb022964ba60e0ab8e515b0/A: closing flushed file at 1733629222582Flushing 9b6311f5dfb022964ba60e0ab8e515b0/B: creating writer at 1733629222992 (+410 ms)Flushing 9b6311f5dfb022964ba60e0ab8e515b0/B: appending metadata at 1733629222995 (+3 ms)Flushing 9b6311f5dfb022964ba60e0ab8e515b0/B: closing flushed file at 1733629222995Flushing 9b6311f5dfb022964ba60e0ab8e515b0/C: creating writer at 1733629223412 (+417 ms)Flushing 9b6311f5dfb022964ba60e0ab8e515b0/C: appending metadata at 1733629223415 (+3 ms)Flushing 9b6311f5dfb022964ba60e0ab8e515b0/C: closing flushed file at 1733629223415Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6120af44: reopening flushed file at 1733629223825 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ad11c4b: reopening flushed file at 1733629223831 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36e49254: reopening flushed file at 1733629223837 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 9b6311f5dfb022964ba60e0ab8e515b0 in 1263ms, sequenceid=574, compaction requested=true at 1733629223842 (+5 ms)Writing region close event to WAL at 1733629223994 (+152 ms)Running coprocessor post-close hooks at 1733629223998 (+4 ms)Closed at 1733629223998 2024-12-08T03:40:24,001 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:24,001 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9b6311f5dfb022964ba60e0ab8e515b0, regionState=CLOSED 2024-12-08T03:40:24,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:40:24,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T03:40:24,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 9b6311f5dfb022964ba60e0ab8e515b0, server=32df4b4bb439,42629,1733629194165 in 2.6690 sec 2024-12-08T03:40:24,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-08T03:40:24,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b6311f5dfb022964ba60e0ab8e515b0, UNASSIGN in 2.6750 sec 2024-12-08T03:40:24,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T03:40:24,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6810 sec 2024-12-08T03:40:24,011 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629224011"}]},"ts":"1733629224011"} 2024-12-08T03:40:24,013 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:40:24,013 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:40:24,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6940 sec 2024-12-08T03:40:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:40:25,499 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:25,503 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:40:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,509 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:40:25,511 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,515 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:25,520 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/recovered.edits] 2024-12-08T03:40:25,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/3f307521552849009896a1f77de97927 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/3f307521552849009896a1f77de97927 2024-12-08T03:40:25,525 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6a6bdd0d687349d4b0922c570a9bc0c1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/6a6bdd0d687349d4b0922c570a9bc0c1 2024-12-08T03:40:25,527 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/77f2301cc9eb4146a4cf34d701587bbf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/A/77f2301cc9eb4146a4cf34d701587bbf 2024-12-08T03:40:25,530 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1365f65c37a14945a7e1627ddb45cf86 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/1365f65c37a14945a7e1627ddb45cf86 2024-12-08T03:40:25,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/3319bec774414b039ccc14ca2b19136e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/3319bec774414b039ccc14ca2b19136e 2024-12-08T03:40:25,535 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/acf39626c6844256872440faac60b333 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/B/acf39626c6844256872440faac60b333 2024-12-08T03:40:25,538 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/49a20db218f44ee983e6ef6d611be4df to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/49a20db218f44ee983e6ef6d611be4df 2024-12-08T03:40:25,540 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5a79f44618df4c038ce4e7bae8bc0f66 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/5a79f44618df4c038ce4e7bae8bc0f66 2024-12-08T03:40:25,542 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c9f3694cec3e48a88805fc2e7d0ee29e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/C/c9f3694cec3e48a88805fc2e7d0ee29e 2024-12-08T03:40:25,545 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/recovered.edits/577.seqid to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0/recovered.edits/577.seqid 2024-12-08T03:40:25,546 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/9b6311f5dfb022964ba60e0ab8e515b0 2024-12-08T03:40:25,546 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:40:25,551 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-08T03:40:25,561 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:40:25,589 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:40:25,591 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,591 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:40:25,591 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733629225591"}]},"ts":"9223372036854775807"} 2024-12-08T03:40:25,595 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:40:25,595 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9b6311f5dfb022964ba60e0ab8e515b0, NAME => 'TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:40:25,595 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:40:25,596 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733629225595"}]},"ts":"9223372036854775807"} 2024-12-08T03:40:25,598 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:40:25,599 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-12-08T03:40:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:40:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:40:25,620 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:25,633 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;32df4b4bb439:42629-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1605528618_22 at /127.0.0.1:57138 [Waiting for operation #260] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557067002_22 at /127.0.0.1:57246 [Waiting for operation #253] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1605528618_22 at /127.0.0.1:34186 [Waiting for operation #240] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=479 (was 309) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17503 (was 18335) 2024-12-08T03:40:25,644 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=17501 2024-12-08T03:40:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:40:25,646 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:40:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:25,648 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:40:25,648 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:25,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-08T03:40:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T03:40:25,649 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:40:25,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741957_1133 (size=959) 2024-12-08T03:40:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T03:40:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T03:40:26,057 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:40:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741958_1134 (size=53) 2024-12-08T03:40:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T03:40:26,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:40:26,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0d57468d599971b9a5cc1f3688630c93, disabling compactions & flushes 2024-12-08T03:40:26,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. after waiting 0 ms 2024-12-08T03:40:26,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d57468d599971b9a5cc1f3688630c93: Waiting for close lock at 1733629226468Disabling compacts and flushes for region at 1733629226468Disabling writes for close at 1733629226468Writing region close event to WAL at 1733629226468Closed at 1733629226468 2024-12-08T03:40:26,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:40:26,470 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733629226470"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733629226470"}]},"ts":"1733629226470"} 2024-12-08T03:40:26,471 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:40:26,472 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:40:26,472 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629226472"}]},"ts":"1733629226472"} 2024-12-08T03:40:26,474 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:40:26,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, ASSIGN}] 2024-12-08T03:40:26,475 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, ASSIGN 2024-12-08T03:40:26,476 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, ASSIGN; state=OFFLINE, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=false 2024-12-08T03:40:26,627 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:26,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, ASSIGN because future has completed 2024-12-08T03:40:26,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:40:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T03:40:26,791 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,791 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:40:26,792 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,792 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:40:26,792 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,792 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,795 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,797 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d57468d599971b9a5cc1f3688630c93 columnFamilyName A 2024-12-08T03:40:26,798 DEBUG [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:26,798 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(327): Store=0d57468d599971b9a5cc1f3688630c93/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:26,798 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,800 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d57468d599971b9a5cc1f3688630c93 columnFamilyName B 2024-12-08T03:40:26,800 DEBUG [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:26,801 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(327): Store=0d57468d599971b9a5cc1f3688630c93/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:26,801 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,802 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d57468d599971b9a5cc1f3688630c93 columnFamilyName C 2024-12-08T03:40:26,802 DEBUG [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:26,803 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(327): Store=0d57468d599971b9a5cc1f3688630c93/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:26,803 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,803 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,804 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,805 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,805 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,806 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:40:26,807 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,808 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:40:26,809 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 0d57468d599971b9a5cc1f3688630c93; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74163423, jitterRate=0.10512112081050873}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:40:26,809 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:26,809 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 0d57468d599971b9a5cc1f3688630c93: Running coprocessor pre-open hook at 1733629226792Writing region info on filesystem at 1733629226792Initializing all the Stores at 1733629226794 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629226794Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629226794Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629226794Cleaning up temporary data from old regions at 1733629226805 (+11 ms)Running coprocessor post-open hooks at 1733629226809 (+4 ms)Region opened successfully at 1733629226809 2024-12-08T03:40:26,810 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., pid=30, masterSystemTime=1733629226784 2024-12-08T03:40:26,812 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,812 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:26,812 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:26,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:40:26,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T03:40:26,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 in 185 msec 2024-12-08T03:40:26,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T03:40:26,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, ASSIGN in 343 msec 2024-12-08T03:40:26,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:40:26,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629226819"}]},"ts":"1733629226819"} 2024-12-08T03:40:26,820 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:40:26,821 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:40:26,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-08T03:40:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T03:40:27,799 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:27,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:27,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:27,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:27,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16525882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:27,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610ba11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:27,803 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:27,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:27,807 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:40:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:40:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@412d21f3 2024-12-08T03:40:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:40:27,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:40:27,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:40:27,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:40:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:27,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741959_1135 (size=995) 2024-12-08T03:40:28,233 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-08T03:40:28,233 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-08T03:40:28,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:40:28,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, REOPEN/MOVE}] 2024-12-08T03:40:28,245 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, REOPEN/MOVE 2024-12-08T03:40:28,246 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, REOPEN/MOVE because future has completed 2024-12-08T03:40:28,248 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:40:28,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:40:28,401 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,401 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:40:28,401 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 0d57468d599971b9a5cc1f3688630c93, disabling compactions & flushes 2024-12-08T03:40:28,401 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,401 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,401 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. after waiting 0 ms 2024-12-08T03:40:28,401 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,405 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T03:40:28,406 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,406 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 0d57468d599971b9a5cc1f3688630c93: Waiting for close lock at 1733629228401Running coprocessor pre-close hooks at 1733629228401Disabling compacts and flushes for region at 1733629228401Disabling writes for close at 1733629228401Writing region close event to WAL at 1733629228401Running coprocessor post-close hooks at 1733629228405 (+4 ms)Closed at 1733629228406 (+1 ms) 2024-12-08T03:40:28,406 WARN [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 0d57468d599971b9a5cc1f3688630c93 to self. 2024-12-08T03:40:28,408 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,409 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=CLOSED 2024-12-08T03:40:28,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:40:28,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-08T03:40:28,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 in 165 msec 2024-12-08T03:40:28,416 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=true 2024-12-08T03:40:28,568 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, REOPEN/MOVE because future has completed 2024-12-08T03:40:28,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:40:28,735 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,736 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:40:28,737 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,737 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:40:28,737 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,738 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,742 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,747 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d57468d599971b9a5cc1f3688630c93 columnFamilyName A 2024-12-08T03:40:28,749 DEBUG [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:28,749 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(327): Store=0d57468d599971b9a5cc1f3688630c93/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:28,750 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,750 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d57468d599971b9a5cc1f3688630c93 columnFamilyName B 2024-12-08T03:40:28,751 DEBUG [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:28,751 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(327): Store=0d57468d599971b9a5cc1f3688630c93/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:28,751 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,752 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d57468d599971b9a5cc1f3688630c93 columnFamilyName C 2024-12-08T03:40:28,752 DEBUG [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:28,753 INFO [StoreOpener-0d57468d599971b9a5cc1f3688630c93-1 {}] regionserver.HStore(327): Store=0d57468d599971b9a5cc1f3688630c93/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:28,753 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,753 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,754 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,755 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,755 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,756 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:40:28,757 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,757 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0d57468d599971b9a5cc1f3688630c93; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72510675, jitterRate=0.0804932564496994}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:40:28,758 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,759 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0d57468d599971b9a5cc1f3688630c93: Running coprocessor pre-open hook at 1733629228738Writing region info on filesystem at 1733629228738Initializing all the Stores at 1733629228739 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629228739Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629228739Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629228739Cleaning up temporary data from old regions at 1733629228755 (+16 ms)Running coprocessor post-open hooks at 1733629228758 (+3 ms)Region opened successfully at 1733629228759 (+1 ms) 2024-12-08T03:40:28,760 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., pid=35, masterSystemTime=1733629228727 2024-12-08T03:40:28,761 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,761 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,762 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=OPEN, openSeqNum=5, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:40:28,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-08T03:40:28,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 in 191 msec 2024-12-08T03:40:28,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T03:40:28,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, REOPEN/MOVE in 521 msec 2024-12-08T03:40:28,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-08T03:40:28,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-12-08T03:40:28,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-12-08T03:40:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T03:40:28,777 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:28,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,778 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28666302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,780 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab3e094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13905c2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,786 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ce4b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,787 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639c2a9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57059a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,791 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1c5d80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,793 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76953a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,794 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5137be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67096705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3403cfa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aed0cf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228997ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eeff5bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f80d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:28,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:28,808 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2889e089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:28,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e7d43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:28,811 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,811 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,811 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,811 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,812 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-08T03:40:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T03:40:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,815 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:28,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,816 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,816 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:28,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,818 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,821 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,827 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:28,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,837 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:28,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:40:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:40:28,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208963a4d65f6d24cfb8b16aa7882286b1e_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629228835/Put/seqid=0 2024-12-08T03:40:28,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741960_1136 (size=14594) 2024-12-08T03:40:28,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:28,903 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208963a4d65f6d24cfb8b16aa7882286b1e_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208963a4d65f6d24cfb8b16aa7882286b1e_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:28,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/32acb33f29e34b14889424ecf159832e, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629288891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:28,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:28,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629288906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629288904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629288907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/32acb33f29e34b14889424ecf159832e is 175, key is test_row_0/A:col10/1733629228835/Put/seqid=0 2024-12-08T03:40:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629288908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T03:40:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:28,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741961_1137 (size=39549) 2024-12-08T03:40:28,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:28,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:28,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:28,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:28,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629289019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629289019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629289019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629289029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629289029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T03:40:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629289228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629289229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629289235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629289239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629289239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,333 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/32acb33f29e34b14889424ecf159832e 2024-12-08T03:40:29,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/2ffb252db2b0469b86a4f4aa2171a60e is 50, key is test_row_0/B:col10/1733629228835/Put/seqid=0 2024-12-08T03:40:29,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741962_1138 (size=12001) 2024-12-08T03:40:29,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:29,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T03:40:29,538 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:40:29,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629289538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629289548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629289548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629289549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629289549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:29,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:29,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:29,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:29,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:29,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:29,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/2ffb252db2b0469b86a4f4aa2171a60e 2024-12-08T03:40:29,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d is 50, key is test_row_0/C:col10/1733629228835/Put/seqid=0 2024-12-08T03:40:29,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741963_1139 (size=12001) 2024-12-08T03:40:29,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d 2024-12-08T03:40:29,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/32acb33f29e34b14889424ecf159832e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e 2024-12-08T03:40:29,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e, entries=200, sequenceid=15, filesize=38.6 K 2024-12-08T03:40:29,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/2ffb252db2b0469b86a4f4aa2171a60e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/2ffb252db2b0469b86a4f4aa2171a60e 2024-12-08T03:40:29,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/2ffb252db2b0469b86a4f4aa2171a60e, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:40:29,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d 2024-12-08T03:40:29,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:40:29,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0d57468d599971b9a5cc1f3688630c93 in 1032ms, sequenceid=15, compaction requested=false 2024-12-08T03:40:29,869 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:40:29,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:29,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-08T03:40:29,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:29,904 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:40:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087d1744a21ce7440c9cdd55fec244e5e9_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629228901/Put/seqid=0 2024-12-08T03:40:29,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741964_1140 (size=12154) 2024-12-08T03:40:29,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:29,940 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087d1744a21ce7440c9cdd55fec244e5e9_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087d1744a21ce7440c9cdd55fec244e5e9_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:29,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/7d8650e1ad21496499e55cf58de927a5, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/7d8650e1ad21496499e55cf58de927a5 is 175, key is test_row_0/A:col10/1733629228901/Put/seqid=0 2024-12-08T03:40:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T03:40:29,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741965_1141 (size=30955) 2024-12-08T03:40:29,951 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/7d8650e1ad21496499e55cf58de927a5 2024-12-08T03:40:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/af2f9cf0c3c0469ba0d4ced7f3f38fce is 50, key is test_row_0/B:col10/1733629228901/Put/seqid=0 2024-12-08T03:40:29,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741966_1142 (size=12001) 2024-12-08T03:40:29,979 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/af2f9cf0c3c0469ba0d4ced7f3f38fce 2024-12-08T03:40:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/0fc8d60963144f7d903be3aa0dfef41c is 50, key is test_row_0/C:col10/1733629228901/Put/seqid=0 2024-12-08T03:40:30,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741967_1143 (size=12001) 2024-12-08T03:40:30,016 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/0fc8d60963144f7d903be3aa0dfef41c 2024-12-08T03:40:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/7d8650e1ad21496499e55cf58de927a5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5 2024-12-08T03:40:30,031 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5, entries=150, sequenceid=40, filesize=30.2 K 2024-12-08T03:40:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/af2f9cf0c3c0469ba0d4ced7f3f38fce as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/af2f9cf0c3c0469ba0d4ced7f3f38fce 2024-12-08T03:40:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,038 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/af2f9cf0c3c0469ba0d4ced7f3f38fce, entries=150, sequenceid=40, filesize=11.7 K 2024-12-08T03:40:30,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/0fc8d60963144f7d903be3aa0dfef41c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0fc8d60963144f7d903be3aa0dfef41c 2024-12-08T03:40:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,048 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0fc8d60963144f7d903be3aa0dfef41c, entries=150, sequenceid=40, filesize=11.7 K 2024-12-08T03:40:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,054 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 0d57468d599971b9a5cc1f3688630c93 in 150ms, sequenceid=40, compaction requested=false 2024-12-08T03:40:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-08T03:40:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-08T03:40:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-08T03:40:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2400 sec 2024-12-08T03:40:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.2480 sec 2024-12-08T03:40:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:30,151 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:30,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:40:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083522639a72364c57b7b4a7dddc5450b3_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:30,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741969_1145 (size=24358) 2024-12-08T03:40:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629290224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629290226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629290226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629290232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629290235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629290339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629290339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629290349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629290349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629290349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629290548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629290548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629290558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629290558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629290558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:30,613 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083522639a72364c57b7b4a7dddc5450b3_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083522639a72364c57b7b4a7dddc5450b3_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:30,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/a108f5dc2cf14c85a51618a39f6b2d8d, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:30,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/a108f5dc2cf14c85a51618a39f6b2d8d is 175, key is test_row_0/A:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741968_1144 (size=73991) 2024-12-08T03:40:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629290859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629290859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629290868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629290869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629290869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T03:40:30,959 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:30,959 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-08T03:40:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-08T03:40:30,965 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:30,967 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:30,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:31,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/a108f5dc2cf14c85a51618a39f6b2d8d 2024-12-08T03:40:31,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/28194e3a11c54063bf18daf6336c028e is 50, key is test_row_0/B:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:31,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741970_1146 (size=9657) 2024-12-08T03:40:31,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/28194e3a11c54063bf18daf6336c028e 2024-12-08T03:40:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-08T03:40:31,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/3437e24ba2fb496a8385856f8968943d is 50, key is test_row_0/C:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:31,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741971_1147 (size=9657) 2024-12-08T03:40:31,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/3437e24ba2fb496a8385856f8968943d 2024-12-08T03:40:31,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/a108f5dc2cf14c85a51618a39f6b2d8d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d 2024-12-08T03:40:31,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d, entries=400, sequenceid=51, filesize=72.3 K 2024-12-08T03:40:31,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/28194e3a11c54063bf18daf6336c028e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/28194e3a11c54063bf18daf6336c028e 2024-12-08T03:40:31,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-08T03:40:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:31,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/28194e3a11c54063bf18daf6336c028e, entries=100, sequenceid=51, filesize=9.4 K 2024-12-08T03:40:31,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/3437e24ba2fb496a8385856f8968943d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/3437e24ba2fb496a8385856f8968943d 2024-12-08T03:40:31,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/3437e24ba2fb496a8385856f8968943d, entries=100, sequenceid=51, filesize=9.4 K 2024-12-08T03:40:31,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0d57468d599971b9a5cc1f3688630c93 in 996ms, sequenceid=51, compaction requested=true 2024-12-08T03:40:31,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:31,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:31,147 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:31,149 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:31,151 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144495 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:31,151 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/A is initiating minor compaction (all files) 2024-12-08T03:40:31,151 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/A in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,151 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=141.1 K 2024-12-08T03:40:31,151 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,152 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d] 2024-12-08T03:40:31,152 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32acb33f29e34b14889424ecf159832e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629228824 2024-12-08T03:40:31,152 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:31,152 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/B is initiating minor compaction (all files) 2024-12-08T03:40:31,152 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/B in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,152 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/2ffb252db2b0469b86a4f4aa2171a60e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/af2f9cf0c3c0469ba0d4ced7f3f38fce, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/28194e3a11c54063bf18daf6336c028e] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=32.9 K 2024-12-08T03:40:31,153 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8650e1ad21496499e55cf58de927a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629228901 2024-12-08T03:40:31,153 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffb252db2b0469b86a4f4aa2171a60e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629228824 2024-12-08T03:40:31,154 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting af2f9cf0c3c0469ba0d4ced7f3f38fce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629228901 2024-12-08T03:40:31,154 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting a108f5dc2cf14c85a51618a39f6b2d8d, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629230094 2024-12-08T03:40:31,154 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 28194e3a11c54063bf18daf6336c028e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629230148 2024-12-08T03:40:31,183 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:31,184 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#B#compaction#129 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:31,186 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/464e329802024de2bb4802668d52c7e4 is 50, key is test_row_0/B:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:31,187 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412083df3c4eb33ce43d78bddd036c2d0dc65_0d57468d599971b9a5cc1f3688630c93 store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:31,192 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412083df3c4eb33ce43d78bddd036c2d0dc65_0d57468d599971b9a5cc1f3688630c93, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:31,192 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083df3c4eb33ce43d78bddd036c2d0dc65_0d57468d599971b9a5cc1f3688630c93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:31,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741973_1149 (size=4469) 2024-12-08T03:40:31,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741972_1148 (size=12104) 2024-12-08T03:40:31,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-08T03:40:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,278 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:40:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-08T03:40:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120869251a1357a34bd496ff758a3f34a63d_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629230229/Put/seqid=0 2024-12-08T03:40:31,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741974_1150 (size=12154) 2024-12-08T03:40:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:31,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:31,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629291379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629291384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629291385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629291386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629291391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629291488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629291499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629291499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629291499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629291499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-08T03:40:31,608 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#A#compaction#130 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:31,609 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/9382c49a1b1c4da1bcee965ece528252 is 175, key is test_row_0/A:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:31,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741975_1151 (size=31165) 2024-12-08T03:40:31,633 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/9382c49a1b1c4da1bcee965ece528252 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/9382c49a1b1c4da1bcee965ece528252 2024-12-08T03:40:31,644 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/464e329802024de2bb4802668d52c7e4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/464e329802024de2bb4802668d52c7e4 2024-12-08T03:40:31,648 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/A of 0d57468d599971b9a5cc1f3688630c93 into 9382c49a1b1c4da1bcee965ece528252(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:31,648 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:31,648 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/A, priority=13, startTime=1733629231147; duration=0sec 2024-12-08T03:40:31,649 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:31,649 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:31,649 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:31,651 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:31,652 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/C is initiating minor compaction (all files) 2024-12-08T03:40:31,652 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/C in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,652 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0fc8d60963144f7d903be3aa0dfef41c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/3437e24ba2fb496a8385856f8968943d] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=32.9 K 2024-12-08T03:40:31,652 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/B of 0d57468d599971b9a5cc1f3688630c93 into 464e329802024de2bb4802668d52c7e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:31,652 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:31,652 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/B, priority=13, startTime=1733629231149; duration=0sec 2024-12-08T03:40:31,652 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d70ad63a5e4cd9ab9a1ba5e10a4b0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629228824 2024-12-08T03:40:31,653 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:31,653 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:31,653 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc8d60963144f7d903be3aa0dfef41c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629228901 2024-12-08T03:40:31,653 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3437e24ba2fb496a8385856f8968943d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629230148 2024-12-08T03:40:31,677 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:31,678 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2 is 50, key is test_row_0/C:col10/1733629230148/Put/seqid=0 2024-12-08T03:40:31,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629291699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:31,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629291708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,714 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120869251a1357a34bd496ff758a3f34a63d_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120869251a1357a34bd496ff758a3f34a63d_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629291709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629291709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/3d4172b037c046269beaf5f80efa7d57, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629291708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/3d4172b037c046269beaf5f80efa7d57 is 175, key is test_row_0/A:col10/1733629230229/Put/seqid=0 2024-12-08T03:40:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:31,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741976_1152 (size=12104) 2024-12-08T03:40:31,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741977_1153 (size=30955) 2024-12-08T03:40:31,761 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/3d4172b037c046269beaf5f80efa7d57 2024-12-08T03:40:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/31a2d7ae61474bee8535d906e81e4e9a is 50, key is test_row_0/B:col10/1733629230229/Put/seqid=0 2024-12-08T03:40:31,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741978_1154 (size=12001) 2024-12-08T03:40:31,795 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/31a2d7ae61474bee8535d906e81e4e9a 2024-12-08T03:40:31,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cb2685cec1504a0086d1a273df15fe53 is 50, key is test_row_0/C:col10/1733629230229/Put/seqid=0 2024-12-08T03:40:31,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741979_1155 (size=12001) 2024-12-08T03:40:31,821 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cb2685cec1504a0086d1a273df15fe53 2024-12-08T03:40:31,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/3d4172b037c046269beaf5f80efa7d57 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57 2024-12-08T03:40:31,837 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57, entries=150, sequenceid=76, filesize=30.2 K 2024-12-08T03:40:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/31a2d7ae61474bee8535d906e81e4e9a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31a2d7ae61474bee8535d906e81e4e9a 2024-12-08T03:40:31,849 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31a2d7ae61474bee8535d906e81e4e9a, entries=150, sequenceid=76, filesize=11.7 K 2024-12-08T03:40:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cb2685cec1504a0086d1a273df15fe53 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cb2685cec1504a0086d1a273df15fe53 2024-12-08T03:40:31,858 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cb2685cec1504a0086d1a273df15fe53, entries=150, sequenceid=76, filesize=11.7 K 2024-12-08T03:40:31,859 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 0d57468d599971b9a5cc1f3688630c93 in 581ms, sequenceid=76, compaction requested=false 2024-12-08T03:40:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-08T03:40:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-08T03:40:31,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-08T03:40:31,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 893 msec 2024-12-08T03:40:31,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 906 msec 2024-12-08T03:40:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:32,011 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:32,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:40:32,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120871b68e425bb744e3a26a640c5531ed3c_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629292048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629292048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629292048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629292052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629292057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741980_1156 (size=12154) 2024-12-08T03:40:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-08T03:40:32,099 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:32,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-08T03:40:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:32,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:32,103 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:32,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:32,133 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2 2024-12-08T03:40:32,141 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/C of 0d57468d599971b9a5cc1f3688630c93 into bb2f8c24e4424ee3a4d0b7ffc7ca4cb2(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:32,141 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:32,141 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/C, priority=13, startTime=1733629231149; duration=0sec 2024-12-08T03:40:32,142 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:32,142 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629292168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629292169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629292169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629292169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629292169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:32,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629292379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629292379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629292379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629292379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629292379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:32,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:32,469 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:32,475 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120871b68e425bb744e3a26a640c5531ed3c_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120871b68e425bb744e3a26a640c5531ed3c_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:32,476 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f078ed1b4bcc48aabc05ec843e19e67a, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:32,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f078ed1b4bcc48aabc05ec843e19e67a is 175, key is test_row_0/A:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741981_1157 (size=30955) 2024-12-08T03:40:32,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:32,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629292688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629292688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629292689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629292689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629292689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:32,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:32,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:32,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:32,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f078ed1b4bcc48aabc05ec843e19e67a 2024-12-08T03:40:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:32,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:32,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/bf704a7c2b9a4ae397dda09091308740 is 50, key is test_row_0/B:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:32,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741982_1158 (size=12001) 2024-12-08T03:40:32,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/bf704a7c2b9a4ae397dda09091308740 2024-12-08T03:40:32,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/99fa8839f509492897ff92c76fab971d is 50, key is test_row_0/C:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:32,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741983_1159 (size=12001) 2024-12-08T03:40:33,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629293198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629293198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629293209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629293210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629293209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:33,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/99fa8839f509492897ff92c76fab971d 2024-12-08T03:40:33,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f078ed1b4bcc48aabc05ec843e19e67a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a 2024-12-08T03:40:33,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a, entries=150, sequenceid=91, filesize=30.2 K 2024-12-08T03:40:33,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/bf704a7c2b9a4ae397dda09091308740 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bf704a7c2b9a4ae397dda09091308740 2024-12-08T03:40:33,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bf704a7c2b9a4ae397dda09091308740, entries=150, sequenceid=91, filesize=11.7 K 2024-12-08T03:40:33,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/99fa8839f509492897ff92c76fab971d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/99fa8839f509492897ff92c76fab971d 2024-12-08T03:40:33,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:33,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/99fa8839f509492897ff92c76fab971d, entries=150, sequenceid=91, filesize=11.7 K 2024-12-08T03:40:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0d57468d599971b9a5cc1f3688630c93 in 1341ms, sequenceid=91, compaction requested=true 2024-12-08T03:40:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:33,353 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:33,353 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:33,355 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:33,355 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/A is initiating minor compaction (all files) 2024-12-08T03:40:33,355 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:33,355 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/A in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,355 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/B is initiating minor compaction (all files) 2024-12-08T03:40:33,355 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/B in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,355 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/9382c49a1b1c4da1bcee965ece528252, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=90.9 K 2024-12-08T03:40:33,355 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/464e329802024de2bb4802668d52c7e4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31a2d7ae61474bee8535d906e81e4e9a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bf704a7c2b9a4ae397dda09091308740] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=35.3 K 2024-12-08T03:40:33,355 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,355 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/9382c49a1b1c4da1bcee965ece528252, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a] 2024-12-08T03:40:33,356 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 464e329802024de2bb4802668d52c7e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629228905 2024-12-08T03:40:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:33,356 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9382c49a1b1c4da1bcee965ece528252, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629228905 2024-12-08T03:40:33,356 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a2d7ae61474bee8535d906e81e4e9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733629230224 2024-12-08T03:40:33,357 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4172b037c046269beaf5f80efa7d57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733629230224 2024-12-08T03:40:33,357 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bf704a7c2b9a4ae397dda09091308740, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629231384 2024-12-08T03:40:33,357 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f078ed1b4bcc48aabc05ec843e19e67a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629231384 2024-12-08T03:40:33,369 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:33,370 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:33,371 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/31217b6c1e1d44e5a8696a8d0f5984a7 is 50, key is test_row_0/B:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:33,375 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412081a6c161770974705b8f1c82019e9812f_0d57468d599971b9a5cc1f3688630c93 store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:33,377 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412081a6c161770974705b8f1c82019e9812f_0d57468d599971b9a5cc1f3688630c93, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:33,377 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081a6c161770974705b8f1c82019e9812f_0d57468d599971b9a5cc1f3688630c93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:33,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741984_1160 (size=12207) 2024-12-08T03:40:33,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741985_1161 (size=4469) 2024-12-08T03:40:33,400 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/31217b6c1e1d44e5a8696a8d0f5984a7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31217b6c1e1d44e5a8696a8d0f5984a7 2024-12-08T03:40:33,405 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/B of 0d57468d599971b9a5cc1f3688630c93 into 31217b6c1e1d44e5a8696a8d0f5984a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:33,405 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:33,405 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/B, priority=13, startTime=1733629233353; duration=0sec 2024-12-08T03:40:33,406 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:33,406 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:33,406 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:33,407 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:33,407 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/C is initiating minor compaction (all files) 2024-12-08T03:40:33,407 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/C in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,407 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cb2685cec1504a0086d1a273df15fe53, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/99fa8839f509492897ff92c76fab971d] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=35.3 K 2024-12-08T03:40:33,407 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bb2f8c24e4424ee3a4d0b7ffc7ca4cb2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629228905 2024-12-08T03:40:33,408 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2685cec1504a0086d1a273df15fe53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733629230224 2024-12-08T03:40:33,409 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 99fa8839f509492897ff92c76fab971d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629231384 2024-12-08T03:40:33,417 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:33,417 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/0f87a4d76bed4c9dbc64d0eff4989dde is 50, key is test_row_0/C:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:33,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741986_1162 (size=12207) 2024-12-08T03:40:33,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:40:33,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:33,505 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:40:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120883497d19f2184369814f90abb75f105c_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629232046/Put/seqid=0 2024-12-08T03:40:33,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741987_1163 (size=12154) 2024-12-08T03:40:33,793 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#A#compaction#138 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:33,793 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/0094091d1f5840b8b73cd0a3330b0848 is 175, key is test_row_0/A:col10/1733629232010/Put/seqid=0 2024-12-08T03:40:33,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741988_1164 (size=31161) 2024-12-08T03:40:33,843 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/0f87a4d76bed4c9dbc64d0eff4989dde as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0f87a4d76bed4c9dbc64d0eff4989dde 2024-12-08T03:40:33,851 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/C of 0d57468d599971b9a5cc1f3688630c93 into 0f87a4d76bed4c9dbc64d0eff4989dde(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:33,851 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:33,851 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/C, priority=13, startTime=1733629233356; duration=0sec 2024-12-08T03:40:33,851 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:33,851 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:33,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:33,925 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120883497d19f2184369814f90abb75f105c_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120883497d19f2184369814f90abb75f105c_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/b9dfe180222343388b7014c3abbdbde2, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:33,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/b9dfe180222343388b7014c3abbdbde2 is 175, key is test_row_0/A:col10/1733629232046/Put/seqid=0 2024-12-08T03:40:33,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741989_1165 (size=30955) 2024-12-08T03:40:34,203 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/0094091d1f5840b8b73cd0a3330b0848 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/0094091d1f5840b8b73cd0a3330b0848 2024-12-08T03:40:34,209 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/A of 0d57468d599971b9a5cc1f3688630c93 into 0094091d1f5840b8b73cd0a3330b0848(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:34,209 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:34,209 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/A, priority=13, startTime=1733629233353; duration=0sec 2024-12-08T03:40:34,209 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:34,209 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:34,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:34,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629294228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,231 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 0d57468d599971b9a5cc1f3688630c93 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629294230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629294230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629294230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629294231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:34,331 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/b9dfe180222343388b7014c3abbdbde2 2024-12-08T03:40:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629294341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629294341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/5bd86daacaaa4051acf9a928b96520e7 is 50, key is test_row_0/B:col10/1733629232046/Put/seqid=0 2024-12-08T03:40:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629294341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629294341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741990_1166 (size=12001) 2024-12-08T03:40:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629294548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629294549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629294549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629294549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,765 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/5bd86daacaaa4051acf9a928b96520e7 2024-12-08T03:40:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/76c492c13ba14c09a80c0f3b9b699ecf is 50, key is test_row_0/C:col10/1733629232046/Put/seqid=0 2024-12-08T03:40:34,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741991_1167 (size=12001) 2024-12-08T03:40:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629294858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629294858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629294858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629294858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,177 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/76c492c13ba14c09a80c0f3b9b699ecf 2024-12-08T03:40:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/b9dfe180222343388b7014c3abbdbde2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2 2024-12-08T03:40:35,185 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2, entries=150, sequenceid=116, filesize=30.2 K 2024-12-08T03:40:35,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/5bd86daacaaa4051acf9a928b96520e7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5bd86daacaaa4051acf9a928b96520e7 2024-12-08T03:40:35,191 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5bd86daacaaa4051acf9a928b96520e7, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T03:40:35,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/76c492c13ba14c09a80c0f3b9b699ecf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76c492c13ba14c09a80c0f3b9b699ecf 2024-12-08T03:40:35,198 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76c492c13ba14c09a80c0f3b9b699ecf, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T03:40:35,199 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 0d57468d599971b9a5cc1f3688630c93 in 1694ms, sequenceid=116, compaction requested=false 2024-12-08T03:40:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-08T03:40:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-08T03:40:35,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-08T03:40:35,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0970 sec 2024-12-08T03:40:35,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.1040 sec 2024-12-08T03:40:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:35,371 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:35,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-08T03:40:35,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120835d7117ebfec4311af504fac60fd5f50_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:35,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741992_1168 (size=14794) 2024-12-08T03:40:35,385 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:35,391 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120835d7117ebfec4311af504fac60fd5f50_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120835d7117ebfec4311af504fac60fd5f50_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:35,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f2ffe3360cc94dfdb6478ef3011f1eb6, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:35,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f2ffe3360cc94dfdb6478ef3011f1eb6 is 175, key is test_row_0/A:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629295395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629295397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629295401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629295402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741993_1169 (size=39749) 2024-12-08T03:40:35,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629295508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629295509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629295509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629295509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629295718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629295719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629295719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629295719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:35,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f2ffe3360cc94dfdb6478ef3011f1eb6 2024-12-08T03:40:35,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/15993f5a164a4b08b5d02e2eee2867ce is 50, key is test_row_0/B:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:35,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741994_1170 (size=12151) 2024-12-08T03:40:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629296028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629296028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629296028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629296028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/15993f5a164a4b08b5d02e2eee2867ce 2024-12-08T03:40:36,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/d31a338b1b6846a4b78a8ec2b17d8762 is 50, key is test_row_0/C:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:36,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741995_1171 (size=12151) 2024-12-08T03:40:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629296248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,250 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 0d57468d599971b9a5cc1f3688630c93 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:40:36,269 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:36,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-08T03:40:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:36,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:36,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:36,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:36,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:40:36,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:36,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:36,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629296538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629296538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629296538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629296538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:36,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:40:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:36,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:36,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/d31a338b1b6846a4b78a8ec2b17d8762 2024-12-08T03:40:36,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/f2ffe3360cc94dfdb6478ef3011f1eb6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6 2024-12-08T03:40:36,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6, entries=200, sequenceid=135, filesize=38.8 K 2024-12-08T03:40:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/15993f5a164a4b08b5d02e2eee2867ce as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/15993f5a164a4b08b5d02e2eee2867ce 2024-12-08T03:40:36,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/15993f5a164a4b08b5d02e2eee2867ce, entries=150, sequenceid=135, filesize=11.9 K 2024-12-08T03:40:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/d31a338b1b6846a4b78a8ec2b17d8762 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/d31a338b1b6846a4b78a8ec2b17d8762 2024-12-08T03:40:36,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/d31a338b1b6846a4b78a8ec2b17d8762, entries=150, sequenceid=135, filesize=11.9 K 2024-12-08T03:40:36,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 0d57468d599971b9a5cc1f3688630c93 in 1288ms, sequenceid=135, compaction requested=true 2024-12-08T03:40:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:36,659 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:36,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:36,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:36,660 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:36,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:36,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:36,661 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:36,661 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/A is initiating minor compaction (all files) 2024-12-08T03:40:36,661 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/A in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,661 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/0094091d1f5840b8b73cd0a3330b0848, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=99.5 K 2024-12-08T03:40:36,661 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,661 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/0094091d1f5840b8b73cd0a3330b0848, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6] 2024-12-08T03:40:36,662 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:36,662 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/B is initiating minor compaction (all files) 2024-12-08T03:40:36,662 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/B in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,662 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31217b6c1e1d44e5a8696a8d0f5984a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5bd86daacaaa4051acf9a928b96520e7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/15993f5a164a4b08b5d02e2eee2867ce] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=35.5 K 2024-12-08T03:40:36,663 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 31217b6c1e1d44e5a8696a8d0f5984a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629231384 2024-12-08T03:40:36,663 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0094091d1f5840b8b73cd0a3330b0848, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629231384 2024-12-08T03:40:36,663 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd86daacaaa4051acf9a928b96520e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733629232026 2024-12-08T03:40:36,663 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9dfe180222343388b7014c3abbdbde2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733629232026 2024-12-08T03:40:36,664 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ffe3360cc94dfdb6478ef3011f1eb6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733629234229 2024-12-08T03:40:36,664 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 15993f5a164a4b08b5d02e2eee2867ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733629235369 2024-12-08T03:40:36,672 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:36,673 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/45e0aa09811e49ce9ea57da7c247f693 is 50, key is test_row_0/B:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:36,673 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:36,685 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208f6933f1d33344efa87c392fa506c867d_0d57468d599971b9a5cc1f3688630c93 store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:36,686 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208f6933f1d33344efa87c392fa506c867d_0d57468d599971b9a5cc1f3688630c93, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:36,686 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f6933f1d33344efa87c392fa506c867d_0d57468d599971b9a5cc1f3688630c93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:36,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741996_1172 (size=12459) 2024-12-08T03:40:36,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741997_1173 (size=4469) 2024-12-08T03:40:36,692 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#A#compaction#148 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:36,693 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/c5e63200236d42f388111eab699ec077 is 175, key is test_row_0/A:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:36,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741998_1174 (size=31413) 2024-12-08T03:40:36,709 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/c5e63200236d42f388111eab699ec077 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/c5e63200236d42f388111eab699ec077 2024-12-08T03:40:36,715 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/A of 0d57468d599971b9a5cc1f3688630c93 into c5e63200236d42f388111eab699ec077(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:36,715 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:36,715 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/A, priority=13, startTime=1733629236659; duration=0sec 2024-12-08T03:40:36,715 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:36,716 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:36,716 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:36,717 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:36,717 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/C is initiating minor compaction (all files) 2024-12-08T03:40:36,717 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/C in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,717 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0f87a4d76bed4c9dbc64d0eff4989dde, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76c492c13ba14c09a80c0f3b9b699ecf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/d31a338b1b6846a4b78a8ec2b17d8762] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=35.5 K 2024-12-08T03:40:36,718 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f87a4d76bed4c9dbc64d0eff4989dde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629231384 2024-12-08T03:40:36,718 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c492c13ba14c09a80c0f3b9b699ecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733629232026 2024-12-08T03:40:36,718 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d31a338b1b6846a4b78a8ec2b17d8762, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733629235369 2024-12-08T03:40:36,726 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:36,727 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/03e106f63f1547d9a09c39cb08218ffb is 50, key is test_row_0/C:col10/1733629235370/Put/seqid=0 2024-12-08T03:40:36,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:40:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:36,731 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-08T03:40:36,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208376c355d8cc64bf883b7668b91baaf6b_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629235393/Put/seqid=0 2024-12-08T03:40:36,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741999_1175 (size=12459) 2024-12-08T03:40:36,757 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/03e106f63f1547d9a09c39cb08218ffb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/03e106f63f1547d9a09c39cb08218ffb 2024-12-08T03:40:36,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742000_1176 (size=12304) 2024-12-08T03:40:36,765 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/C of 0d57468d599971b9a5cc1f3688630c93 into 03e106f63f1547d9a09c39cb08218ffb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:36,765 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:36,765 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/C, priority=13, startTime=1733629236660; duration=0sec 2024-12-08T03:40:36,766 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:36,766 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:37,102 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/45e0aa09811e49ce9ea57da7c247f693 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/45e0aa09811e49ce9ea57da7c247f693 2024-12-08T03:40:37,107 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/B of 0d57468d599971b9a5cc1f3688630c93 into 45e0aa09811e49ce9ea57da7c247f693(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:37,107 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:37,107 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/B, priority=13, startTime=1733629236660; duration=0sec 2024-12-08T03:40:37,107 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:37,107 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:37,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:37,166 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208376c355d8cc64bf883b7668b91baaf6b_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208376c355d8cc64bf883b7668b91baaf6b_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:37,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/2741b5cd95aa411e94ad5df53bdeb544, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:37,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/2741b5cd95aa411e94ad5df53bdeb544 is 175, key is test_row_0/A:col10/1733629235393/Put/seqid=0 2024-12-08T03:40:37,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742001_1177 (size=31105) 2024-12-08T03:40:37,224 INFO [master/32df4b4bb439:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T03:40:37,224 INFO [master/32df4b4bb439:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T03:40:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:37,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:37,573 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/2741b5cd95aa411e94ad5df53bdeb544 2024-12-08T03:40:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629297569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629297570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629297571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629297573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/ae4e6cebd9f54d83a70a49889693ce51 is 50, key is test_row_0/B:col10/1733629235393/Put/seqid=0 2024-12-08T03:40:37,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742002_1178 (size=12151) 2024-12-08T03:40:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629297678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629297678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629297678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629297688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629297888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629297889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629297889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629297898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:37,993 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/ae4e6cebd9f54d83a70a49889693ce51 2024-12-08T03:40:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cc2cb1fd95a54cc68595dec6830257b7 is 50, key is test_row_0/C:col10/1733629235393/Put/seqid=0 2024-12-08T03:40:38,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742003_1179 (size=12151) 2024-12-08T03:40:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629298198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629298198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629298198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629298208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,407 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cc2cb1fd95a54cc68595dec6830257b7 2024-12-08T03:40:38,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/2741b5cd95aa411e94ad5df53bdeb544 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544 2024-12-08T03:40:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:38,421 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544, entries=150, sequenceid=155, filesize=30.4 K 2024-12-08T03:40:38,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/ae4e6cebd9f54d83a70a49889693ce51 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/ae4e6cebd9f54d83a70a49889693ce51 2024-12-08T03:40:38,427 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/ae4e6cebd9f54d83a70a49889693ce51, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:40:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cc2cb1fd95a54cc68595dec6830257b7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cc2cb1fd95a54cc68595dec6830257b7 2024-12-08T03:40:38,432 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cc2cb1fd95a54cc68595dec6830257b7, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:40:38,433 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 0d57468d599971b9a5cc1f3688630c93 in 1703ms, sequenceid=155, compaction requested=false 2024-12-08T03:40:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-08T03:40:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-08T03:40:38,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-08T03:40:38,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-12-08T03:40:38,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1670 sec 2024-12-08T03:40:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:38,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-08T03:40:38,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087c69d9d23daf411c9b5d7ae2eb6d25f4_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:38,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742004_1180 (size=12304) 2024-12-08T03:40:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629298726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629298727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629298731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629298731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629298838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629298838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629298838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629298838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629299048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629299048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629299048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629299048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:39,126 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087c69d9d23daf411c9b5d7ae2eb6d25f4_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087c69d9d23daf411c9b5d7ae2eb6d25f4_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:39,127 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/eb008e1f7d2b4a1ab105a25326d5c97b, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:39,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/eb008e1f7d2b4a1ab105a25326d5c97b is 175, key is test_row_0/A:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742005_1181 (size=31105) 2024-12-08T03:40:39,132 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/eb008e1f7d2b4a1ab105a25326d5c97b 2024-12-08T03:40:39,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/930c871e8ad041f1a8ee52e2fb672316 is 50, key is test_row_0/B:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:39,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742006_1182 (size=12151) 2024-12-08T03:40:39,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629299358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629299359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629299359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629299359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/930c871e8ad041f1a8ee52e2fb672316 2024-12-08T03:40:39,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/8cb272271b0d49358d8fd3d23a71b365 is 50, key is test_row_0/C:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:39,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742007_1183 (size=12151) 2024-12-08T03:40:39,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/8cb272271b0d49358d8fd3d23a71b365 2024-12-08T03:40:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/eb008e1f7d2b4a1ab105a25326d5c97b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b 2024-12-08T03:40:39,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b, entries=150, sequenceid=176, filesize=30.4 K 2024-12-08T03:40:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/930c871e8ad041f1a8ee52e2fb672316 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/930c871e8ad041f1a8ee52e2fb672316 2024-12-08T03:40:39,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/930c871e8ad041f1a8ee52e2fb672316, entries=150, sequenceid=176, filesize=11.9 K 2024-12-08T03:40:39,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/8cb272271b0d49358d8fd3d23a71b365 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/8cb272271b0d49358d8fd3d23a71b365 2024-12-08T03:40:39,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/8cb272271b0d49358d8fd3d23a71b365, entries=150, sequenceid=176, filesize=11.9 K 2024-12-08T03:40:39,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 0d57468d599971b9a5cc1f3688630c93 in 867ms, sequenceid=176, compaction requested=true 2024-12-08T03:40:39,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:39,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:39,578 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:39,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:39,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:39,578 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:39,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:39,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:39,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/B is initiating minor compaction (all files) 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/A is initiating minor compaction (all files) 2024-12-08T03:40:39,579 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/B in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:39,579 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/A in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:39,579 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/45e0aa09811e49ce9ea57da7c247f693, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/ae4e6cebd9f54d83a70a49889693ce51, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/930c871e8ad041f1a8ee52e2fb672316] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=35.9 K 2024-12-08T03:40:39,579 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/c5e63200236d42f388111eab699ec077, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=91.4 K 2024-12-08T03:40:39,579 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/c5e63200236d42f388111eab699ec077, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b] 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e0aa09811e49ce9ea57da7c247f693, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733629235369 2024-12-08T03:40:39,579 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e63200236d42f388111eab699ec077, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733629235369 2024-12-08T03:40:39,580 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4e6cebd9f54d83a70a49889693ce51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733629235393 2024-12-08T03:40:39,580 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2741b5cd95aa411e94ad5df53bdeb544, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733629235393 2024-12-08T03:40:39,580 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 930c871e8ad041f1a8ee52e2fb672316, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733629237568 2024-12-08T03:40:39,580 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb008e1f7d2b4a1ab105a25326d5c97b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733629237568 2024-12-08T03:40:39,588 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:39,589 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/bc29b2d0f4b44467b04760f0e02485c3 is 50, key is test_row_0/B:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:39,589 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:39,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742008_1184 (size=12561) 2024-12-08T03:40:39,594 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412084c32ebfb85bf40048af3c8b7130b3d31_0d57468d599971b9a5cc1f3688630c93 store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:39,595 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412084c32ebfb85bf40048af3c8b7130b3d31_0d57468d599971b9a5cc1f3688630c93, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:39,595 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084c32ebfb85bf40048af3c8b7130b3d31_0d57468d599971b9a5cc1f3688630c93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:39,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742009_1185 (size=4469) 2024-12-08T03:40:39,609 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#A#compaction#157 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:39,610 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/fcd02da30170445aac9a5b5481c0304b is 175, key is test_row_0/A:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742010_1186 (size=31515) 2024-12-08T03:40:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:39,871 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:39,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-08T03:40:39,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208afa0ce4e16444837ae6c8f39e00800ab_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629239871/Put/seqid=0 2024-12-08T03:40:39,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742011_1187 (size=12304) 2024-12-08T03:40:39,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:39,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208afa0ce4e16444837ae6c8f39e00800ab_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208afa0ce4e16444837ae6c8f39e00800ab_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:39,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/12eac42ce33e4b29a2d67c87fc53e644, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:39,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/12eac42ce33e4b29a2d67c87fc53e644 is 175, key is test_row_0/A:col10/1733629239871/Put/seqid=0 2024-12-08T03:40:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742012_1188 (size=31105) 2024-12-08T03:40:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629299891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629299917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629299917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629299917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:39,999 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/bc29b2d0f4b44467b04760f0e02485c3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bc29b2d0f4b44467b04760f0e02485c3 2024-12-08T03:40:40,007 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/B of 0d57468d599971b9a5cc1f3688630c93 into bc29b2d0f4b44467b04760f0e02485c3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:40,007 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:40,007 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/B, priority=13, startTime=1733629239578; duration=0sec 2024-12-08T03:40:40,007 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:40,007 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:40,007 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:40,008 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:40,008 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/C is initiating minor compaction (all files) 2024-12-08T03:40:40,008 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/C in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:40,009 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/03e106f63f1547d9a09c39cb08218ffb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cc2cb1fd95a54cc68595dec6830257b7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/8cb272271b0d49358d8fd3d23a71b365] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=35.9 K 2024-12-08T03:40:40,009 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e106f63f1547d9a09c39cb08218ffb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733629235369 2024-12-08T03:40:40,009 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2cb1fd95a54cc68595dec6830257b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733629235393 2024-12-08T03:40:40,009 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb272271b0d49358d8fd3d23a71b365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733629237568 2024-12-08T03:40:40,017 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:40,018 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cf4acfbe59e04431bb9d92b7515bd5bb is 50, key is test_row_0/C:col10/1733629237568/Put/seqid=0 2024-12-08T03:40:40,021 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/fcd02da30170445aac9a5b5481c0304b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/fcd02da30170445aac9a5b5481c0304b 2024-12-08T03:40:40,027 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/A of 0d57468d599971b9a5cc1f3688630c93 into fcd02da30170445aac9a5b5481c0304b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:40,027 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:40,027 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/A, priority=13, startTime=1733629239578; duration=0sec 2024-12-08T03:40:40,027 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:40,027 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:40,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629300028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629300028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629300029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629300029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742013_1189 (size=12561) 2024-12-08T03:40:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629300238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629300238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629300238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629300238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54800 deadline: 1733629300269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,270 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 0d57468d599971b9a5cc1f3688630c93 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:40:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/12eac42ce33e4b29a2d67c87fc53e644 2024-12-08T03:40:40,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/477cf78c777e4d928d9fe38ab7a09b47 is 50, key is test_row_0/B:col10/1733629239871/Put/seqid=0 2024-12-08T03:40:40,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742014_1190 (size=12151) 2024-12-08T03:40:40,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/477cf78c777e4d928d9fe38ab7a09b47 2024-12-08T03:40:40,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/06936ad9017349d681799d84c2b81b09 is 50, key is test_row_0/C:col10/1733629239871/Put/seqid=0 2024-12-08T03:40:40,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742015_1191 (size=12151) 2024-12-08T03:40:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:40:40,438 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:40,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-08T03:40:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:40:40,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:40,442 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:40,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:40,442 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/cf4acfbe59e04431bb9d92b7515bd5bb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cf4acfbe59e04431bb9d92b7515bd5bb 2024-12-08T03:40:40,449 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/C of 0d57468d599971b9a5cc1f3688630c93 into cf4acfbe59e04431bb9d92b7515bd5bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:40,449 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:40,449 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/C, priority=13, startTime=1733629239578; duration=0sec 2024-12-08T03:40:40,449 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:40,449 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:40:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629300548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629300548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629300548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629300548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:40,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:40:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:40,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:40,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/06936ad9017349d681799d84c2b81b09 2024-12-08T03:40:40,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/12eac42ce33e4b29a2d67c87fc53e644 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644 2024-12-08T03:40:40,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644, entries=150, sequenceid=196, filesize=30.4 K 2024-12-08T03:40:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/477cf78c777e4d928d9fe38ab7a09b47 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/477cf78c777e4d928d9fe38ab7a09b47 2024-12-08T03:40:40,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/477cf78c777e4d928d9fe38ab7a09b47, entries=150, sequenceid=196, filesize=11.9 K 2024-12-08T03:40:40,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/06936ad9017349d681799d84c2b81b09 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/06936ad9017349d681799d84c2b81b09 2024-12-08T03:40:40,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:40:40,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:40,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:40,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:40,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/06936ad9017349d681799d84c2b81b09, entries=150, sequenceid=196, filesize=11.9 K 2024-12-08T03:40:40,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0d57468d599971b9a5cc1f3688630c93 in 883ms, sequenceid=196, compaction requested=false 2024-12-08T03:40:40,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:40:40,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:40:40,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:40,900 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-08T03:40:40,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120807b1c9be4c824ed2930e6b48caa6963d_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629239898/Put/seqid=0 2024-12-08T03:40:40,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742016_1192 (size=12304) 2024-12-08T03:40:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:40:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629301073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629301075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629301075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629301077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629301188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629301188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629301188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629301188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:41,353 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120807b1c9be4c824ed2930e6b48caa6963d_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120807b1c9be4c824ed2930e6b48caa6963d_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:41,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/1ce685cae8634c5da4434cc9a159e675, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:41,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/1ce685cae8634c5da4434cc9a159e675 is 175, key is test_row_0/A:col10/1733629239898/Put/seqid=0 2024-12-08T03:40:41,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742017_1193 (size=31105) 2024-12-08T03:40:41,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629301398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629301398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629301398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629301398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:40:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629301708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629301708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629301708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629301708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:41,758 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/1ce685cae8634c5da4434cc9a159e675 2024-12-08T03:40:41,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/9387938df3d34c26905262096bdaf6ae is 50, key is test_row_0/B:col10/1733629239898/Put/seqid=0 2024-12-08T03:40:41,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742018_1194 (size=12151) 2024-12-08T03:40:42,171 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/9387938df3d34c26905262096bdaf6ae 2024-12-08T03:40:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/579a75cdcec147f287fb929dcdbe1a26 is 50, key is test_row_0/C:col10/1733629239898/Put/seqid=0 2024-12-08T03:40:42,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742019_1195 (size=12151) 2024-12-08T03:40:42,183 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/579a75cdcec147f287fb929dcdbe1a26 2024-12-08T03:40:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/1ce685cae8634c5da4434cc9a159e675 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675 2024-12-08T03:40:42,192 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675, entries=150, sequenceid=215, filesize=30.4 K 2024-12-08T03:40:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/9387938df3d34c26905262096bdaf6ae as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/9387938df3d34c26905262096bdaf6ae 2024-12-08T03:40:42,198 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/9387938df3d34c26905262096bdaf6ae, entries=150, sequenceid=215, filesize=11.9 K 2024-12-08T03:40:42,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/579a75cdcec147f287fb929dcdbe1a26 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/579a75cdcec147f287fb929dcdbe1a26 2024-12-08T03:40:42,203 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/579a75cdcec147f287fb929dcdbe1a26, entries=150, sequenceid=215, filesize=11.9 K 2024-12-08T03:40:42,204 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0d57468d599971b9a5cc1f3688630c93 in 1305ms, sequenceid=215, compaction requested=true 2024-12-08T03:40:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-08T03:40:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-08T03:40:42,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-08T03:40:42,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7630 sec 2024-12-08T03:40:42,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7690 sec 2024-12-08T03:40:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:42,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:42,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-08T03:40:42,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208559c4358a87c41b9a24cf246ef4a73e2_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742020_1196 (size=14794) 2024-12-08T03:40:42,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:42,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629302235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208559c4358a87c41b9a24cf246ef4a73e2_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208559c4358a87c41b9a24cf246ef4a73e2_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629302236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d59a9c218655422396ea992bd92dd6d7, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629302237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629302238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d59a9c218655422396ea992bd92dd6d7 is 175, key is test_row_0/A:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742021_1197 (size=39749) 2024-12-08T03:40:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629302348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629302348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629302348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629302348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629302558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629302558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629302558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629302558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:40:42,588 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:42,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-08T03:40:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:42,590 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:42,591 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:42,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:42,648 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d59a9c218655422396ea992bd92dd6d7 2024-12-08T03:40:42,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/4126b36fca6d4a5ba323787b39c7c486 is 50, key is test_row_0/B:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:42,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742022_1198 (size=12151) 2024-12-08T03:40:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:42,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:40:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:42,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:42,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629302868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629302868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629302868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629302868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:42,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:40:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:42,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:43,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:40:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/4126b36fca6d4a5ba323787b39c7c486 2024-12-08T03:40:43,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/4d648846c7bb4459be8c600375b8bce8 is 50, key is test_row_0/C:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:43,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742023_1199 (size=12151) 2024-12-08T03:40:43,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:40:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:43,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:40:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629303379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:43,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629303379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629303379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:43,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629303388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:43,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/4d648846c7bb4459be8c600375b8bce8 2024-12-08T03:40:43,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d59a9c218655422396ea992bd92dd6d7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7 2024-12-08T03:40:43,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7, entries=200, sequenceid=238, filesize=38.8 K 2024-12-08T03:40:43,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/4126b36fca6d4a5ba323787b39c7c486 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/4126b36fca6d4a5ba323787b39c7c486 2024-12-08T03:40:43,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/4126b36fca6d4a5ba323787b39c7c486, entries=150, sequenceid=238, filesize=11.9 K 2024-12-08T03:40:43,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/4d648846c7bb4459be8c600375b8bce8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/4d648846c7bb4459be8c600375b8bce8 2024-12-08T03:40:43,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/4d648846c7bb4459be8c600375b8bce8, entries=150, sequenceid=238, filesize=11.9 K 2024-12-08T03:40:43,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0d57468d599971b9a5cc1f3688630c93 in 1268ms, sequenceid=238, compaction requested=true 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:43,492 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:43,492 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:43,493 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:43,493 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/A is initiating minor compaction (all files) 2024-12-08T03:40:43,493 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:43,493 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/B is initiating minor compaction (all files) 2024-12-08T03:40:43,493 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/A in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,493 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/B in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,493 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bc29b2d0f4b44467b04760f0e02485c3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/477cf78c777e4d928d9fe38ab7a09b47, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/9387938df3d34c26905262096bdaf6ae, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/4126b36fca6d4a5ba323787b39c7c486] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=47.9 K 2024-12-08T03:40:43,493 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/fcd02da30170445aac9a5b5481c0304b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=130.3 K 2024-12-08T03:40:43,493 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,494 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/fcd02da30170445aac9a5b5481c0304b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7] 2024-12-08T03:40:43,494 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcd02da30170445aac9a5b5481c0304b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733629237568 2024-12-08T03:40:43,494 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bc29b2d0f4b44467b04760f0e02485c3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733629237568 2024-12-08T03:40:43,494 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12eac42ce33e4b29a2d67c87fc53e644, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733629238730 2024-12-08T03:40:43,494 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 477cf78c777e4d928d9fe38ab7a09b47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733629238730 2024-12-08T03:40:43,495 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce685cae8634c5da4434cc9a159e675, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733629239882 2024-12-08T03:40:43,495 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 9387938df3d34c26905262096bdaf6ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733629239882 2024-12-08T03:40:43,495 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d59a9c218655422396ea992bd92dd6d7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629241075 2024-12-08T03:40:43,495 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 4126b36fca6d4a5ba323787b39c7c486, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629242221 2024-12-08T03:40:43,501 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:43,502 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:43,503 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/0514fb3471184b47a19f2f556c4f5f5a is 50, key is test_row_0/B:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:43,506 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120833343841b72d471995e27cc88489eda2_0d57468d599971b9a5cc1f3688630c93 store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:43,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:40:43,508 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120833343841b72d471995e27cc88489eda2_0d57468d599971b9a5cc1f3688630c93, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:43,508 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120833343841b72d471995e27cc88489eda2_0d57468d599971b9a5cc1f3688630c93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,508 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:40:43,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742024_1200 (size=12697) 2024-12-08T03:40:43,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742025_1201 (size=4469) 2024-12-08T03:40:43,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120861516186c5124850ae7903398fd4cbb8_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629242237/Put/seqid=0 2024-12-08T03:40:43,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742026_1202 (size=12304) 2024-12-08T03:40:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:43,925 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:43,926 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/de57d32015a24d4398f1929298bb1c98 is 175, key is test_row_0/A:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:43,927 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/0514fb3471184b47a19f2f556c4f5f5a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0514fb3471184b47a19f2f556c4f5f5a 2024-12-08T03:40:43,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742027_1203 (size=31651) 2024-12-08T03:40:43,932 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/B of 0d57468d599971b9a5cc1f3688630c93 into 0514fb3471184b47a19f2f556c4f5f5a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:43,932 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:43,932 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/B, priority=12, startTime=1733629243492; duration=0sec 2024-12-08T03:40:43,933 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:43,933 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:43,933 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:40:43,934 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:40:43,934 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/C is initiating minor compaction (all files) 2024-12-08T03:40:43,934 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/C in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:43,934 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cf4acfbe59e04431bb9d92b7515bd5bb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/06936ad9017349d681799d84c2b81b09, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/579a75cdcec147f287fb929dcdbe1a26, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/4d648846c7bb4459be8c600375b8bce8] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=47.9 K 2024-12-08T03:40:43,935 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/de57d32015a24d4398f1929298bb1c98 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/de57d32015a24d4398f1929298bb1c98 2024-12-08T03:40:43,935 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cf4acfbe59e04431bb9d92b7515bd5bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733629237568 2024-12-08T03:40:43,935 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 06936ad9017349d681799d84c2b81b09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733629238730 2024-12-08T03:40:43,936 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 579a75cdcec147f287fb929dcdbe1a26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733629239882 2024-12-08T03:40:43,936 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d648846c7bb4459be8c600375b8bce8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629242221 2024-12-08T03:40:43,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:43,941 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120861516186c5124850ae7903398fd4cbb8_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861516186c5124850ae7903398fd4cbb8_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:43,941 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/A of 0d57468d599971b9a5cc1f3688630c93 into de57d32015a24d4398f1929298bb1c98(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:43,941 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:43,942 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/A, priority=12, startTime=1733629243492; duration=0sec 2024-12-08T03:40:43,942 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:43,942 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:43,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/38b7921810294fe9b77176200f2f4d4d, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/38b7921810294fe9b77176200f2f4d4d is 175, key is test_row_0/A:col10/1733629242237/Put/seqid=0 2024-12-08T03:40:43,946 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:43,947 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/2954a06a3c4942849923406593b06411 is 50, key is test_row_0/C:col10/1733629242222/Put/seqid=0 2024-12-08T03:40:43,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742028_1204 (size=31105) 2024-12-08T03:40:43,950 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/38b7921810294fe9b77176200f2f4d4d 2024-12-08T03:40:43,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742029_1205 (size=12697) 2024-12-08T03:40:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/1b1eb5ba265744c0a6a7dd481ae13af1 is 50, key is test_row_0/B:col10/1733629242237/Put/seqid=0 2024-12-08T03:40:43,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742030_1206 (size=12151) 2024-12-08T03:40:44,359 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/2954a06a3c4942849923406593b06411 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2954a06a3c4942849923406593b06411 2024-12-08T03:40:44,364 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/C of 0d57468d599971b9a5cc1f3688630c93 into 2954a06a3c4942849923406593b06411(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:44,364 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:44,364 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/C, priority=12, startTime=1733629243492; duration=0sec 2024-12-08T03:40:44,364 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:44,364 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:44,366 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/1b1eb5ba265744c0a6a7dd481ae13af1 2024-12-08T03:40:44,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/59770b03f8b042f5bd30646185eb9818 is 50, key is test_row_0/C:col10/1733629242237/Put/seqid=0 2024-12-08T03:40:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:44,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:44,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742031_1207 (size=12151) 2024-12-08T03:40:44,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629304412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629304413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629304414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629304415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629304518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629304528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629304528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629304528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629304728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629304738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629304738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629304738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:44,796 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/59770b03f8b042f5bd30646185eb9818 2024-12-08T03:40:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/38b7921810294fe9b77176200f2f4d4d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d 2024-12-08T03:40:44,805 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d, entries=150, sequenceid=251, filesize=30.4 K 2024-12-08T03:40:44,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/1b1eb5ba265744c0a6a7dd481ae13af1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/1b1eb5ba265744c0a6a7dd481ae13af1 2024-12-08T03:40:44,811 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/1b1eb5ba265744c0a6a7dd481ae13af1, entries=150, sequenceid=251, filesize=11.9 K 2024-12-08T03:40:44,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/59770b03f8b042f5bd30646185eb9818 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/59770b03f8b042f5bd30646185eb9818 2024-12-08T03:40:44,816 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/59770b03f8b042f5bd30646185eb9818, entries=150, sequenceid=251, filesize=11.9 K 2024-12-08T03:40:44,817 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0d57468d599971b9a5cc1f3688630c93 in 1309ms, sequenceid=251, compaction requested=false 2024-12-08T03:40:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-08T03:40:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-08T03:40:44,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-08T03:40:44,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2280 sec 2024-12-08T03:40:44,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2320 sec 2024-12-08T03:40:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:45,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:45,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:40:45,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ccb25183f1c54eab9375e6d818926a1c_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742032_1208 (size=14994) 2024-12-08T03:40:45,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629305056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629305057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629305058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629305058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629305168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629305168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629305168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629305168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629305378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629305379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629305379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629305379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:45,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ccb25183f1c54eab9375e6d818926a1c_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ccb25183f1c54eab9375e6d818926a1c_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:45,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/78cb4ab1c87643529816874e6544f8dc, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:45,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/78cb4ab1c87643529816874e6544f8dc is 175, key is test_row_0/A:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742033_1209 (size=39949) 2024-12-08T03:40:45,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629305688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629305688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629305688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629305689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:45,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/78cb4ab1c87643529816874e6544f8dc 2024-12-08T03:40:45,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/f321cfadb00e4b82ba463a203bee4903 is 50, key is test_row_0/B:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742034_1210 (size=12301) 2024-12-08T03:40:45,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/f321cfadb00e4b82ba463a203bee4903 2024-12-08T03:40:45,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/76def34289d94d42b780c7cc28321582 is 50, key is test_row_0/C:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742035_1211 (size=12301) 2024-12-08T03:40:45,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/76def34289d94d42b780c7cc28321582 2024-12-08T03:40:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/78cb4ab1c87643529816874e6544f8dc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc 2024-12-08T03:40:45,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc, entries=200, sequenceid=278, filesize=39.0 K 2024-12-08T03:40:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/f321cfadb00e4b82ba463a203bee4903 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/f321cfadb00e4b82ba463a203bee4903 2024-12-08T03:40:45,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/f321cfadb00e4b82ba463a203bee4903, entries=150, sequenceid=278, filesize=12.0 K 2024-12-08T03:40:45,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/76def34289d94d42b780c7cc28321582 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76def34289d94d42b780c7cc28321582 2024-12-08T03:40:45,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76def34289d94d42b780c7cc28321582, entries=150, sequenceid=278, filesize=12.0 K 2024-12-08T03:40:45,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 0d57468d599971b9a5cc1f3688630c93 in 866ms, sequenceid=278, compaction requested=true 2024-12-08T03:40:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:45,907 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:45,907 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/A is initiating minor compaction (all files) 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/B is initiating minor compaction (all files) 2024-12-08T03:40:45,908 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/A in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:45,908 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/B in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:45,908 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/de57d32015a24d4398f1929298bb1c98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=100.3 K 2024-12-08T03:40:45,908 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0514fb3471184b47a19f2f556c4f5f5a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/1b1eb5ba265744c0a6a7dd481ae13af1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/f321cfadb00e4b82ba463a203bee4903] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=36.3 K 2024-12-08T03:40:45,908 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/de57d32015a24d4398f1929298bb1c98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc] 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0514fb3471184b47a19f2f556c4f5f5a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629242221 2024-12-08T03:40:45,908 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting de57d32015a24d4398f1929298bb1c98, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629242221 2024-12-08T03:40:45,909 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1eb5ba265744c0a6a7dd481ae13af1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733629242230 2024-12-08T03:40:45,909 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b7921810294fe9b77176200f2f4d4d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733629242230 2024-12-08T03:40:45,909 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f321cfadb00e4b82ba463a203bee4903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733629244411 2024-12-08T03:40:45,909 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78cb4ab1c87643529816874e6544f8dc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733629244411 2024-12-08T03:40:45,914 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:45,915 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:45,916 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/047e17b87da4421e848cabed736e32b7 is 50, key is test_row_0/B:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,917 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208bd42636d99fc4f508a65b66bb48d0245_0d57468d599971b9a5cc1f3688630c93 store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:45,918 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208bd42636d99fc4f508a65b66bb48d0245_0d57468d599971b9a5cc1f3688630c93, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:45,918 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bd42636d99fc4f508a65b66bb48d0245_0d57468d599971b9a5cc1f3688630c93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:45,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742036_1212 (size=12949) 2024-12-08T03:40:45,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742037_1213 (size=4469) 2024-12-08T03:40:45,924 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#A#compaction#178 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:45,924 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/722a5c30b56e42239f784b15d0267041 is 175, key is test_row_0/A:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,927 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/047e17b87da4421e848cabed736e32b7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/047e17b87da4421e848cabed736e32b7 2024-12-08T03:40:45,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742038_1214 (size=31903) 2024-12-08T03:40:45,935 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/B of 0d57468d599971b9a5cc1f3688630c93 into 047e17b87da4421e848cabed736e32b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:45,935 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:45,935 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/B, priority=13, startTime=1733629245906; duration=0sec 2024-12-08T03:40:45,935 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:45,935 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:45,935 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:40:45,936 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:40:45,936 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 0d57468d599971b9a5cc1f3688630c93/C is initiating minor compaction (all files) 2024-12-08T03:40:45,936 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d57468d599971b9a5cc1f3688630c93/C in TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:45,936 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2954a06a3c4942849923406593b06411, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/59770b03f8b042f5bd30646185eb9818, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76def34289d94d42b780c7cc28321582] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp, totalSize=36.3 K 2024-12-08T03:40:45,937 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 2954a06a3c4942849923406593b06411, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629242221 2024-12-08T03:40:45,937 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 59770b03f8b042f5bd30646185eb9818, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733629242230 2024-12-08T03:40:45,937 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 76def34289d94d42b780c7cc28321582, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733629244411 2024-12-08T03:40:45,943 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d57468d599971b9a5cc1f3688630c93#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:40:45,944 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/609e12d36ae24983b1c872476afe9bb7 is 50, key is test_row_0/C:col10/1733629245038/Put/seqid=0 2024-12-08T03:40:45,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742039_1215 (size=12949) 2024-12-08T03:40:45,953 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/609e12d36ae24983b1c872476afe9bb7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/609e12d36ae24983b1c872476afe9bb7 2024-12-08T03:40:45,960 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/C of 0d57468d599971b9a5cc1f3688630c93 into 609e12d36ae24983b1c872476afe9bb7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:45,960 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:45,960 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/C, priority=13, startTime=1733629245907; duration=0sec 2024-12-08T03:40:45,960 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:45,960 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:46,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:46,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-08T03:40:46,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120879c20d66e2f2458cba4177641395b1f3_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629246200/Put/seqid=0 2024-12-08T03:40:46,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742040_1216 (size=12454) 2024-12-08T03:40:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629306219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629306220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629306221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629306222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629306328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629306328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629306328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629306338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,378 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/722a5c30b56e42239f784b15d0267041 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/722a5c30b56e42239f784b15d0267041 2024-12-08T03:40:46,383 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d57468d599971b9a5cc1f3688630c93/A of 0d57468d599971b9a5cc1f3688630c93 into 722a5c30b56e42239f784b15d0267041(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:40:46,383 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:46,383 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., storeName=0d57468d599971b9a5cc1f3688630c93/A, priority=13, startTime=1733629245906; duration=0sec 2024-12-08T03:40:46,383 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:46,383 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629306538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629306538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629306538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629306548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:46,621 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120879c20d66e2f2458cba4177641395b1f3_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120879c20d66e2f2458cba4177641395b1f3_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:46,622 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:46,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87 is 175, key is test_row_0/A:col10/1733629246200/Put/seqid=0 2024-12-08T03:40:46,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742041_1217 (size=31255) 2024-12-08T03:40:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:40:46,768 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:46,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-08T03:40:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:46,770 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:46,771 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:46,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629306848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629306849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629306849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629306858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:46,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:46,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:46,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:46,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:46,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:46,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87 2024-12-08T03:40:47,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/baf8ba2f173e4340991e9276d60ed0cf is 50, key is test_row_0/B:col10/1733629246200/Put/seqid=0 2024-12-08T03:40:47,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742042_1218 (size=12301) 2024-12-08T03:40:47,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:47,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:47,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629307358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629307358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629307358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629307368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:47,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:47,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/baf8ba2f173e4340991e9276d60ed0cf 2024-12-08T03:40:47,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/b3e797be59cf4f4ba0d1729a9ef3e5f0 is 50, key is test_row_0/C:col10/1733629246200/Put/seqid=0 2024-12-08T03:40:47,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742043_1219 (size=12301) 2024-12-08T03:40:47,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:47,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/b3e797be59cf4f4ba0d1729a9ef3e5f0 2024-12-08T03:40:47,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87 2024-12-08T03:40:47,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87, entries=150, sequenceid=298, filesize=30.5 K 2024-12-08T03:40:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/baf8ba2f173e4340991e9276d60ed0cf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/baf8ba2f173e4340991e9276d60ed0cf 2024-12-08T03:40:47,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/baf8ba2f173e4340991e9276d60ed0cf, entries=150, sequenceid=298, filesize=12.0 K 2024-12-08T03:40:47,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/b3e797be59cf4f4ba0d1729a9ef3e5f0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/b3e797be59cf4f4ba0d1729a9ef3e5f0 2024-12-08T03:40:47,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/b3e797be59cf4f4ba0d1729a9ef3e5f0, entries=150, sequenceid=298, filesize=12.0 K 2024-12-08T03:40:47,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0d57468d599971b9a5cc1f3688630c93 in 1672ms, sequenceid=298, compaction requested=false 2024-12-08T03:40:47,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:47,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:40:47,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:47,993 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-08T03:40:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120853ae9589fee045cd97ddf5aeb6ca7cbb_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629246221/Put/seqid=0 2024-12-08T03:40:48,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742044_1220 (size=12454) 2024-12-08T03:40:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. as already flushing 2024-12-08T03:40:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629308385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629308385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629308388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629308386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:48,408 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120853ae9589fee045cd97ddf5aeb6ca7cbb_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853ae9589fee045cd97ddf5aeb6ca7cbb_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/422c4a22dfcc4cfbb9077d01a3205203, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/422c4a22dfcc4cfbb9077d01a3205203 is 175, key is test_row_0/A:col10/1733629246221/Put/seqid=0 2024-12-08T03:40:48,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742045_1221 (size=31255) 2024-12-08T03:40:48,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629308498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629308498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629308499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629308499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629308708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629308708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629308708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629308709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:48,820 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/422c4a22dfcc4cfbb9077d01a3205203 2024-12-08T03:40:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/c4503ad2edfa45368d4ee9bcbd73b66e is 50, key is test_row_0/B:col10/1733629246221/Put/seqid=0 2024-12-08T03:40:48,829 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-12-08T03:40:48,829 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:48,829 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,829 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:48,829 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-12-08T03:40:48,829 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:48,829 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:48,830 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,830 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-12-08T03:40:48,830 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:48,830 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,831 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:48,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742046_1222 (size=12301) 2024-12-08T03:40:48,833 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-12-08T03:40:48,833 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:48,833 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,833 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:48,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629309019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629309019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629309019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629309019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,232 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/c4503ad2edfa45368d4ee9bcbd73b66e 2024-12-08T03:40:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/2fdc8a9096204098b79da12a7f747334 is 50, key is test_row_0/C:col10/1733629246221/Put/seqid=0 2024-12-08T03:40:49,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742047_1223 (size=12301) 2024-12-08T03:40:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1733629309529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54786 deadline: 1733629309529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54780 deadline: 1733629309529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54758 deadline: 1733629309529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:40:49,652 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/2fdc8a9096204098b79da12a7f747334 2024-12-08T03:40:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/422c4a22dfcc4cfbb9077d01a3205203 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/422c4a22dfcc4cfbb9077d01a3205203 2024-12-08T03:40:49,669 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/422c4a22dfcc4cfbb9077d01a3205203, entries=150, sequenceid=317, filesize=30.5 K 2024-12-08T03:40:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/c4503ad2edfa45368d4ee9bcbd73b66e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/c4503ad2edfa45368d4ee9bcbd73b66e 2024-12-08T03:40:49,675 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/c4503ad2edfa45368d4ee9bcbd73b66e, entries=150, sequenceid=317, filesize=12.0 K 2024-12-08T03:40:49,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/2fdc8a9096204098b79da12a7f747334 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2fdc8a9096204098b79da12a7f747334 2024-12-08T03:40:49,681 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2fdc8a9096204098b79da12a7f747334, entries=150, sequenceid=317, filesize=12.0 K 2024-12-08T03:40:49,682 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0d57468d599971b9a5cc1f3688630c93 in 1689ms, sequenceid=317, compaction requested=true 2024-12-08T03:40:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-08T03:40:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-08T03:40:49,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-08T03:40:49,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9130 sec 2024-12-08T03:40:49,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.9180 sec 2024-12-08T03:40:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:50,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-08T03:40:50,320 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-12-08T03:40:50,321 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:50,321 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,321 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:50,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ba18a18a187948bfb2427b76e0815536_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629248386/Put/seqid=0 2024-12-08T03:40:50,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742048_1224 (size=12454) 2024-12-08T03:40:50,542 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-12-08T03:40:50,542 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-12-08T03:40:50,542 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:50,542 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:50,542 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,542 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,542 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,542 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:50,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:50,550 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-12-08T03:40:50,550 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-12-08T03:40:50,550 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:50,550 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,550 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:40:50,550 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,550 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:50,550 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:50,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:50,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ba18a18a187948bfb2427b76e0815536_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ba18a18a187948bfb2427b76e0815536_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:50,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e8488c6a428542b485bf748abc74a5e1, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:50,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e8488c6a428542b485bf748abc74a5e1 is 175, key is test_row_0/A:col10/1733629248386/Put/seqid=0 2024-12-08T03:40:50,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742049_1225 (size=31255) 2024-12-08T03:40:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:40:50,940 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4795 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4701 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2084 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6252 rows 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2076 2024-12-08T03:40:50,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6226 rows 2024-12-08T03:40:50,941 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:40:50,941 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:40:50,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:50,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:40:50,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:40:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:50,949 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629250949"}]},"ts":"1733629250949"} 2024-12-08T03:40:50,951 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:40:50,951 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:40:50,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:40:50,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, UNASSIGN}] 2024-12-08T03:40:50,954 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, UNASSIGN 2024-12-08T03:40:50,955 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:50,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, UNASSIGN because future has completed 2024-12-08T03:40:50,956 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:40:50,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:40:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:51,110 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:51,110 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:40:51,110 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 0d57468d599971b9a5cc1f3688630c93, disabling compactions & flushes 2024-12-08T03:40:51,110 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:51,155 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e8488c6a428542b485bf748abc74a5e1 2024-12-08T03:40:51,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/0e76163a022749a59b5754132e39387d is 50, key is test_row_0/B:col10/1733629248386/Put/seqid=0 2024-12-08T03:40:51,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742050_1226 (size=12301) 2024-12-08T03:40:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:51,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/0e76163a022749a59b5754132e39387d 2024-12-08T03:40:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:51,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/31bc0e5ba55d4baea178cfc12d085a5e is 50, key is test_row_0/C:col10/1733629248386/Put/seqid=0 2024-12-08T03:40:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742051_1227 (size=12301) 2024-12-08T03:40:52,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/31bc0e5ba55d4baea178cfc12d085a5e 2024-12-08T03:40:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e8488c6a428542b485bf748abc74a5e1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e8488c6a428542b485bf748abc74a5e1 2024-12-08T03:40:52,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e8488c6a428542b485bf748abc74a5e1, entries=150, sequenceid=336, filesize=30.5 K 2024-12-08T03:40:52,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/0e76163a022749a59b5754132e39387d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0e76163a022749a59b5754132e39387d 2024-12-08T03:40:52,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0e76163a022749a59b5754132e39387d, entries=150, sequenceid=336, filesize=12.0 K 2024-12-08T03:40:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/31bc0e5ba55d4baea178cfc12d085a5e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/31bc0e5ba55d4baea178cfc12d085a5e 2024-12-08T03:40:52,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/31bc0e5ba55d4baea178cfc12d085a5e, entries=150, sequenceid=336, filesize=12.0 K 2024-12-08T03:40:52,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=26.84 KB/27480 for 0d57468d599971b9a5cc1f3688630c93 in 1734ms, sequenceid=336, compaction requested=true 2024-12-08T03:40:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d57468d599971b9a5cc1f3688630c93: 2024-12-08T03:40:52,054 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:52,054 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:52,054 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. after waiting 0 ms 2024-12-08T03:40:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:40:52,054 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:52,054 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 0d57468d599971b9a5cc1f3688630c93 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-08T03:40:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:40:52,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:40:52,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d57468d599971b9a5cc1f3688630c93:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:40:52,055 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. because compaction request was cancelled 2024-12-08T03:40:52,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:40:52,055 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:A 2024-12-08T03:40:52,055 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. because compaction request was cancelled 2024-12-08T03:40:52,055 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. because compaction request was cancelled 2024-12-08T03:40:52,055 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:C 2024-12-08T03:40:52,055 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d57468d599971b9a5cc1f3688630c93:B 2024-12-08T03:40:52,059 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088065dfa25f734871b5a510799fa41460_0d57468d599971b9a5cc1f3688630c93 is 50, key is test_row_0/A:col10/1733629250548/Put/seqid=0 2024-12-08T03:40:52,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742052_1228 (size=12454) 2024-12-08T03:40:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:52,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:40:52,465 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:52,476 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088065dfa25f734871b5a510799fa41460_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088065dfa25f734871b5a510799fa41460_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:52,477 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e30725085c434345bfd561cd6c51b897, store: [table=TestAcidGuarantees family=A region=0d57468d599971b9a5cc1f3688630c93] 2024-12-08T03:40:52,478 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e30725085c434345bfd561cd6c51b897 is 175, key is test_row_0/A:col10/1733629250548/Put/seqid=0 2024-12-08T03:40:52,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742053_1229 (size=31255) 2024-12-08T03:40:52,884 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e30725085c434345bfd561cd6c51b897 2024-12-08T03:40:52,898 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/5ef3387122ef4465979aa8ee99bbf004 is 50, key is test_row_0/B:col10/1733629250548/Put/seqid=0 2024-12-08T03:40:52,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742054_1230 (size=12301) 2024-12-08T03:40:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:53,304 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/5ef3387122ef4465979aa8ee99bbf004 2024-12-08T03:40:53,318 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/de85ff2d9f3b41c9bbfbe2c0a912c08c is 50, key is test_row_0/C:col10/1733629250548/Put/seqid=0 2024-12-08T03:40:53,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742055_1231 (size=12301) 2024-12-08T03:40:53,725 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/de85ff2d9f3b41c9bbfbe2c0a912c08c 2024-12-08T03:40:53,735 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/A/e30725085c434345bfd561cd6c51b897 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e30725085c434345bfd561cd6c51b897 2024-12-08T03:40:53,739 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e30725085c434345bfd561cd6c51b897, entries=150, sequenceid=343, filesize=30.5 K 2024-12-08T03:40:53,740 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/B/5ef3387122ef4465979aa8ee99bbf004 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5ef3387122ef4465979aa8ee99bbf004 2024-12-08T03:40:53,745 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5ef3387122ef4465979aa8ee99bbf004, entries=150, sequenceid=343, filesize=12.0 K 2024-12-08T03:40:53,746 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/.tmp/C/de85ff2d9f3b41c9bbfbe2c0a912c08c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/de85ff2d9f3b41c9bbfbe2c0a912c08c 2024-12-08T03:40:53,750 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/de85ff2d9f3b41c9bbfbe2c0a912c08c, entries=150, sequenceid=343, filesize=12.0 K 2024-12-08T03:40:53,751 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0d57468d599971b9a5cc1f3688630c93 in 1697ms, sequenceid=343, compaction requested=true 2024-12-08T03:40:53,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/9382c49a1b1c4da1bcee965ece528252, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/0094091d1f5840b8b73cd0a3330b0848, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/c5e63200236d42f388111eab699ec077, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/fcd02da30170445aac9a5b5481c0304b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/de57d32015a24d4398f1929298bb1c98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc] to archive 2024-12-08T03:40:53,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:40:53,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/32acb33f29e34b14889424ecf159832e 2024-12-08T03:40:53,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/7d8650e1ad21496499e55cf58de927a5 2024-12-08T03:40:53,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/a108f5dc2cf14c85a51618a39f6b2d8d 2024-12-08T03:40:53,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/9382c49a1b1c4da1bcee965ece528252 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/9382c49a1b1c4da1bcee965ece528252 2024-12-08T03:40:53,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/3d4172b037c046269beaf5f80efa7d57 2024-12-08T03:40:53,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/0094091d1f5840b8b73cd0a3330b0848 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/0094091d1f5840b8b73cd0a3330b0848 2024-12-08T03:40:53,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f078ed1b4bcc48aabc05ec843e19e67a 2024-12-08T03:40:53,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/b9dfe180222343388b7014c3abbdbde2 2024-12-08T03:40:53,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/f2ffe3360cc94dfdb6478ef3011f1eb6 2024-12-08T03:40:53,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/c5e63200236d42f388111eab699ec077 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/c5e63200236d42f388111eab699ec077 2024-12-08T03:40:53,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/2741b5cd95aa411e94ad5df53bdeb544 2024-12-08T03:40:53,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/fcd02da30170445aac9a5b5481c0304b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/fcd02da30170445aac9a5b5481c0304b 2024-12-08T03:40:53,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/eb008e1f7d2b4a1ab105a25326d5c97b 2024-12-08T03:40:53,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/12eac42ce33e4b29a2d67c87fc53e644 2024-12-08T03:40:53,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/1ce685cae8634c5da4434cc9a159e675 2024-12-08T03:40:53,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d59a9c218655422396ea992bd92dd6d7 2024-12-08T03:40:53,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/de57d32015a24d4398f1929298bb1c98 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/de57d32015a24d4398f1929298bb1c98 2024-12-08T03:40:53,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/38b7921810294fe9b77176200f2f4d4d 2024-12-08T03:40:53,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/78cb4ab1c87643529816874e6544f8dc 2024-12-08T03:40:53,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/2ffb252db2b0469b86a4f4aa2171a60e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/af2f9cf0c3c0469ba0d4ced7f3f38fce, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/464e329802024de2bb4802668d52c7e4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/28194e3a11c54063bf18daf6336c028e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31a2d7ae61474bee8535d906e81e4e9a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31217b6c1e1d44e5a8696a8d0f5984a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bf704a7c2b9a4ae397dda09091308740, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5bd86daacaaa4051acf9a928b96520e7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/45e0aa09811e49ce9ea57da7c247f693, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/15993f5a164a4b08b5d02e2eee2867ce, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/ae4e6cebd9f54d83a70a49889693ce51, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bc29b2d0f4b44467b04760f0e02485c3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/930c871e8ad041f1a8ee52e2fb672316, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/477cf78c777e4d928d9fe38ab7a09b47, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/9387938df3d34c26905262096bdaf6ae, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0514fb3471184b47a19f2f556c4f5f5a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/4126b36fca6d4a5ba323787b39c7c486, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/1b1eb5ba265744c0a6a7dd481ae13af1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/f321cfadb00e4b82ba463a203bee4903] to archive 2024-12-08T03:40:53,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:40:53,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/2ffb252db2b0469b86a4f4aa2171a60e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/2ffb252db2b0469b86a4f4aa2171a60e 2024-12-08T03:40:53,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/af2f9cf0c3c0469ba0d4ced7f3f38fce to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/af2f9cf0c3c0469ba0d4ced7f3f38fce 2024-12-08T03:40:53,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/464e329802024de2bb4802668d52c7e4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/464e329802024de2bb4802668d52c7e4 2024-12-08T03:40:53,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/28194e3a11c54063bf18daf6336c028e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/28194e3a11c54063bf18daf6336c028e 2024-12-08T03:40:53,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31a2d7ae61474bee8535d906e81e4e9a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31a2d7ae61474bee8535d906e81e4e9a 2024-12-08T03:40:53,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31217b6c1e1d44e5a8696a8d0f5984a7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/31217b6c1e1d44e5a8696a8d0f5984a7 2024-12-08T03:40:53,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bf704a7c2b9a4ae397dda09091308740 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bf704a7c2b9a4ae397dda09091308740 2024-12-08T03:40:53,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5bd86daacaaa4051acf9a928b96520e7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5bd86daacaaa4051acf9a928b96520e7 2024-12-08T03:40:53,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/45e0aa09811e49ce9ea57da7c247f693 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/45e0aa09811e49ce9ea57da7c247f693 2024-12-08T03:40:53,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/15993f5a164a4b08b5d02e2eee2867ce to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/15993f5a164a4b08b5d02e2eee2867ce 2024-12-08T03:40:53,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/ae4e6cebd9f54d83a70a49889693ce51 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/ae4e6cebd9f54d83a70a49889693ce51 2024-12-08T03:40:53,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bc29b2d0f4b44467b04760f0e02485c3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/bc29b2d0f4b44467b04760f0e02485c3 2024-12-08T03:40:53,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/930c871e8ad041f1a8ee52e2fb672316 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/930c871e8ad041f1a8ee52e2fb672316 2024-12-08T03:40:53,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/477cf78c777e4d928d9fe38ab7a09b47 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/477cf78c777e4d928d9fe38ab7a09b47 2024-12-08T03:40:53,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/9387938df3d34c26905262096bdaf6ae to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/9387938df3d34c26905262096bdaf6ae 2024-12-08T03:40:53,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0514fb3471184b47a19f2f556c4f5f5a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0514fb3471184b47a19f2f556c4f5f5a 2024-12-08T03:40:53,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/4126b36fca6d4a5ba323787b39c7c486 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/4126b36fca6d4a5ba323787b39c7c486 2024-12-08T03:40:53,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/1b1eb5ba265744c0a6a7dd481ae13af1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/1b1eb5ba265744c0a6a7dd481ae13af1 2024-12-08T03:40:53,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/f321cfadb00e4b82ba463a203bee4903 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/f321cfadb00e4b82ba463a203bee4903 2024-12-08T03:40:53,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0fc8d60963144f7d903be3aa0dfef41c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/3437e24ba2fb496a8385856f8968943d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cb2685cec1504a0086d1a273df15fe53, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0f87a4d76bed4c9dbc64d0eff4989dde, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/99fa8839f509492897ff92c76fab971d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76c492c13ba14c09a80c0f3b9b699ecf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/03e106f63f1547d9a09c39cb08218ffb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/d31a338b1b6846a4b78a8ec2b17d8762, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cc2cb1fd95a54cc68595dec6830257b7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cf4acfbe59e04431bb9d92b7515bd5bb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/8cb272271b0d49358d8fd3d23a71b365, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/06936ad9017349d681799d84c2b81b09, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/579a75cdcec147f287fb929dcdbe1a26, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2954a06a3c4942849923406593b06411, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/4d648846c7bb4459be8c600375b8bce8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/59770b03f8b042f5bd30646185eb9818, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76def34289d94d42b780c7cc28321582] to archive 2024-12-08T03:40:53,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:40:53,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/19d70ad63a5e4cd9ab9a1ba5e10a4b0d 2024-12-08T03:40:53,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0fc8d60963144f7d903be3aa0dfef41c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0fc8d60963144f7d903be3aa0dfef41c 2024-12-08T03:40:53,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/bb2f8c24e4424ee3a4d0b7ffc7ca4cb2 2024-12-08T03:40:53,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/3437e24ba2fb496a8385856f8968943d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/3437e24ba2fb496a8385856f8968943d 2024-12-08T03:40:53,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cb2685cec1504a0086d1a273df15fe53 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cb2685cec1504a0086d1a273df15fe53 2024-12-08T03:40:53,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0f87a4d76bed4c9dbc64d0eff4989dde to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/0f87a4d76bed4c9dbc64d0eff4989dde 2024-12-08T03:40:53,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/99fa8839f509492897ff92c76fab971d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/99fa8839f509492897ff92c76fab971d 2024-12-08T03:40:53,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76c492c13ba14c09a80c0f3b9b699ecf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76c492c13ba14c09a80c0f3b9b699ecf 2024-12-08T03:40:53,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/03e106f63f1547d9a09c39cb08218ffb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/03e106f63f1547d9a09c39cb08218ffb 2024-12-08T03:40:53,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/d31a338b1b6846a4b78a8ec2b17d8762 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/d31a338b1b6846a4b78a8ec2b17d8762 2024-12-08T03:40:53,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cc2cb1fd95a54cc68595dec6830257b7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cc2cb1fd95a54cc68595dec6830257b7 2024-12-08T03:40:53,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cf4acfbe59e04431bb9d92b7515bd5bb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/cf4acfbe59e04431bb9d92b7515bd5bb 2024-12-08T03:40:53,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/8cb272271b0d49358d8fd3d23a71b365 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/8cb272271b0d49358d8fd3d23a71b365 2024-12-08T03:40:53,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/06936ad9017349d681799d84c2b81b09 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/06936ad9017349d681799d84c2b81b09 2024-12-08T03:40:53,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/579a75cdcec147f287fb929dcdbe1a26 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/579a75cdcec147f287fb929dcdbe1a26 2024-12-08T03:40:53,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2954a06a3c4942849923406593b06411 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2954a06a3c4942849923406593b06411 2024-12-08T03:40:53,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/4d648846c7bb4459be8c600375b8bce8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/4d648846c7bb4459be8c600375b8bce8 2024-12-08T03:40:53,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/59770b03f8b042f5bd30646185eb9818 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/59770b03f8b042f5bd30646185eb9818 2024-12-08T03:40:53,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76def34289d94d42b780c7cc28321582 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/76def34289d94d42b780c7cc28321582 2024-12-08T03:40:53,839 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=4 2024-12-08T03:40:53,840 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. 2024-12-08T03:40:53,840 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 0d57468d599971b9a5cc1f3688630c93: Waiting for close lock at 1733629251110Running coprocessor pre-close hooks at 1733629251110Disabling compacts and flushes for region at 1733629251110Disabling writes for close at 1733629252054 (+944 ms)Obtaining lock to block concurrent updates at 1733629252055 (+1 ms)Preparing flush snapshotting stores in 0d57468d599971b9a5cc1f3688630c93 at 1733629252055Finished memstore snapshotting TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733629252055Flushing stores of TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93. at 1733629252055Flushing 0d57468d599971b9a5cc1f3688630c93/A: creating writer at 1733629252055Flushing mob file 0d57468d599971b9a5cc1f3688630c93/A: appending metadata at 1733629252059 (+4 ms)Flushing mob file 0d57468d599971b9a5cc1f3688630c93/A: closing flushed file at 1733629252059Flushing 0d57468d599971b9a5cc1f3688630c93/A: appending metadata at 1733629252477 (+418 ms)Flushing 0d57468d599971b9a5cc1f3688630c93/A: closing flushed file at 1733629252477Flushing 0d57468d599971b9a5cc1f3688630c93/B: creating writer at 1733629252894 (+417 ms)Flushing 0d57468d599971b9a5cc1f3688630c93/B: appending metadata at 1733629252897 (+3 ms)Flushing 0d57468d599971b9a5cc1f3688630c93/B: closing flushed file at 1733629252897Flushing 0d57468d599971b9a5cc1f3688630c93/C: creating writer at 1733629253313 (+416 ms)Flushing 0d57468d599971b9a5cc1f3688630c93/C: appending metadata at 1733629253317 (+4 ms)Flushing 0d57468d599971b9a5cc1f3688630c93/C: closing flushed file at 1733629253317Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e5858f: reopening flushed file at 1733629253734 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ee4d808: reopening flushed file at 1733629253739 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558fc84: reopening flushed file at 1733629253745 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0d57468d599971b9a5cc1f3688630c93 in 1697ms, sequenceid=343, compaction requested=true at 1733629253751 (+6 ms)Writing region close event to WAL at 1733629253836 (+85 ms)Running coprocessor post-close hooks at 1733629253840 (+4 ms)Closed at 1733629253840 2024-12-08T03:40:53,842 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:53,843 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0d57468d599971b9a5cc1f3688630c93, regionState=CLOSED 2024-12-08T03:40:53,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:40:53,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-08T03:40:53,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d57468d599971b9a5cc1f3688630c93, server=32df4b4bb439,42629,1733629194165 in 2.8880 sec 2024-12-08T03:40:53,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-08T03:40:53,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d57468d599971b9a5cc1f3688630c93, UNASSIGN in 2.8930 sec 2024-12-08T03:40:53,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-08T03:40:53,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8960 sec 2024-12-08T03:40:53,850 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629253850"}]},"ts":"1733629253850"} 2024-12-08T03:40:53,851 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:40:53,851 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:40:53,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9070 sec 2024-12-08T03:40:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:40:55,110 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:55,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:40:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,116 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:40:55,119 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,124 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,128 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/recovered.edits] 2024-12-08T03:40:55,133 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/422c4a22dfcc4cfbb9077d01a3205203 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/422c4a22dfcc4cfbb9077d01a3205203 2024-12-08T03:40:55,135 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/722a5c30b56e42239f784b15d0267041 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/722a5c30b56e42239f784b15d0267041 2024-12-08T03:40:55,137 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/d9d15c04b6d34a8a8f4e2bf86f0eeb87 2024-12-08T03:40:55,140 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e30725085c434345bfd561cd6c51b897 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e30725085c434345bfd561cd6c51b897 2024-12-08T03:40:55,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e8488c6a428542b485bf748abc74a5e1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/A/e8488c6a428542b485bf748abc74a5e1 2024-12-08T03:40:55,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/047e17b87da4421e848cabed736e32b7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/047e17b87da4421e848cabed736e32b7 2024-12-08T03:40:55,151 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0e76163a022749a59b5754132e39387d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/0e76163a022749a59b5754132e39387d 2024-12-08T03:40:55,154 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5ef3387122ef4465979aa8ee99bbf004 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/5ef3387122ef4465979aa8ee99bbf004 2024-12-08T03:40:55,157 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/baf8ba2f173e4340991e9276d60ed0cf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/baf8ba2f173e4340991e9276d60ed0cf 2024-12-08T03:40:55,159 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/c4503ad2edfa45368d4ee9bcbd73b66e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/B/c4503ad2edfa45368d4ee9bcbd73b66e 2024-12-08T03:40:55,162 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2fdc8a9096204098b79da12a7f747334 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/2fdc8a9096204098b79da12a7f747334 2024-12-08T03:40:55,163 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/31bc0e5ba55d4baea178cfc12d085a5e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/31bc0e5ba55d4baea178cfc12d085a5e 2024-12-08T03:40:55,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/609e12d36ae24983b1c872476afe9bb7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/609e12d36ae24983b1c872476afe9bb7 2024-12-08T03:40:55,165 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/b3e797be59cf4f4ba0d1729a9ef3e5f0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/b3e797be59cf4f4ba0d1729a9ef3e5f0 2024-12-08T03:40:55,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/de85ff2d9f3b41c9bbfbe2c0a912c08c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/C/de85ff2d9f3b41c9bbfbe2c0a912c08c 2024-12-08T03:40:55,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/recovered.edits/346.seqid to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93/recovered.edits/346.seqid 2024-12-08T03:40:55,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:40:55,168 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:40:55,169 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T03:40:55,173 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120807b1c9be4c824ed2930e6b48caa6963d_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120807b1c9be4c824ed2930e6b48caa6963d_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,174 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083522639a72364c57b7b4a7dddc5450b3_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083522639a72364c57b7b4a7dddc5450b3_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,175 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120835d7117ebfec4311af504fac60fd5f50_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120835d7117ebfec4311af504fac60fd5f50_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,177 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208376c355d8cc64bf883b7668b91baaf6b_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208376c355d8cc64bf883b7668b91baaf6b_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,178 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853ae9589fee045cd97ddf5aeb6ca7cbb_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853ae9589fee045cd97ddf5aeb6ca7cbb_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,179 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208559c4358a87c41b9a24cf246ef4a73e2_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208559c4358a87c41b9a24cf246ef4a73e2_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,181 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861516186c5124850ae7903398fd4cbb8_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861516186c5124850ae7903398fd4cbb8_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,182 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120869251a1357a34bd496ff758a3f34a63d_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120869251a1357a34bd496ff758a3f34a63d_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,183 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120871b68e425bb744e3a26a640c5531ed3c_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120871b68e425bb744e3a26a640c5531ed3c_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,184 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120879c20d66e2f2458cba4177641395b1f3_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120879c20d66e2f2458cba4177641395b1f3_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,185 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087c69d9d23daf411c9b5d7ae2eb6d25f4_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087c69d9d23daf411c9b5d7ae2eb6d25f4_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,187 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087d1744a21ce7440c9cdd55fec244e5e9_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087d1744a21ce7440c9cdd55fec244e5e9_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,188 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088065dfa25f734871b5a510799fa41460_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088065dfa25f734871b5a510799fa41460_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,189 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120883497d19f2184369814f90abb75f105c_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120883497d19f2184369814f90abb75f105c_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,190 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208963a4d65f6d24cfb8b16aa7882286b1e_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208963a4d65f6d24cfb8b16aa7882286b1e_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,191 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208afa0ce4e16444837ae6c8f39e00800ab_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208afa0ce4e16444837ae6c8f39e00800ab_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,192 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ba18a18a187948bfb2427b76e0815536_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ba18a18a187948bfb2427b76e0815536_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,193 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ccb25183f1c54eab9375e6d818926a1c_0d57468d599971b9a5cc1f3688630c93 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ccb25183f1c54eab9375e6d818926a1c_0d57468d599971b9a5cc1f3688630c93 2024-12-08T03:40:55,193 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:40:55,195 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,197 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:40:55,199 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:40:55,200 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,200 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:40:55,200 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733629255200"}]},"ts":"9223372036854775807"} 2024-12-08T03:40:55,202 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:40:55,202 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0d57468d599971b9a5cc1f3688630c93, NAME => 'TestAcidGuarantees,,1733629225645.0d57468d599971b9a5cc1f3688630c93.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:40:55,202 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:40:55,202 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733629255202"}]},"ts":"9223372036854775807"} 2024-12-08T03:40:55,204 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:40:55,205 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-12-08T03:40:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:40:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:40:55,229 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:55,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557067002_22 at /127.0.0.1:52764 [Waiting for operation #189] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1605528618_22 at /127.0.0.1:52716 [Waiting for operation #203] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557067002_22 at /127.0.0.1:52780 [Waiting for operation #184] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1605528618_22 at /127.0.0.1:34186 [Waiting for operation #458] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 479), ProcessCount=11 (was 11), AvailableMemoryMB=17252 (was 17501) 2024-12-08T03:40:55,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=465, ProcessCount=11, AvailableMemoryMB=17252 2024-12-08T03:40:55,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:40:55,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:40:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:40:55,248 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:40:55,248 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:55,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-08T03:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-08T03:40:55,249 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:40:55,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742056_1232 (size=959) 2024-12-08T03:40:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-08T03:40:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-08T03:40:55,661 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:40:55,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742057_1233 (size=53) 2024-12-08T03:40:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-08T03:40:56,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:40:56,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8e27db14233afe9225314bddd051f0ac, disabling compactions & flushes 2024-12-08T03:40:56,074 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. after waiting 0 ms 2024-12-08T03:40:56,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,075 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8e27db14233afe9225314bddd051f0ac: Waiting for close lock at 1733629256074Disabling compacts and flushes for region at 1733629256074Disabling writes for close at 1733629256075 (+1 ms)Writing region close event to WAL at 1733629256075Closed at 1733629256075 2024-12-08T03:40:56,077 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:40:56,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733629256077"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733629256077"}]},"ts":"1733629256077"} 2024-12-08T03:40:56,082 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:40:56,083 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:40:56,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629256083"}]},"ts":"1733629256083"} 2024-12-08T03:40:56,086 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:40:56,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, ASSIGN}] 2024-12-08T03:40:56,087 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, ASSIGN 2024-12-08T03:40:56,089 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, ASSIGN; state=OFFLINE, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=false 2024-12-08T03:40:56,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=8e27db14233afe9225314bddd051f0ac, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:56,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, ASSIGN because future has completed 2024-12-08T03:40:56,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:40:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-08T03:40:56,398 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,398 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:40:56,398 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,399 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:40:56,399 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,399 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,400 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,402 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e27db14233afe9225314bddd051f0ac columnFamilyName A 2024-12-08T03:40:56,402 DEBUG [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:56,403 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] regionserver.HStore(327): Store=8e27db14233afe9225314bddd051f0ac/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:56,403 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,406 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e27db14233afe9225314bddd051f0ac columnFamilyName B 2024-12-08T03:40:56,406 DEBUG [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:56,406 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] regionserver.HStore(327): Store=8e27db14233afe9225314bddd051f0ac/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:56,406 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,408 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e27db14233afe9225314bddd051f0ac columnFamilyName C 2024-12-08T03:40:56,408 DEBUG [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:40:56,408 INFO [StoreOpener-8e27db14233afe9225314bddd051f0ac-1 {}] regionserver.HStore(327): Store=8e27db14233afe9225314bddd051f0ac/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:40:56,408 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,409 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,409 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,410 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,410 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,411 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:40:56,412 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,413 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:40:56,414 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 8e27db14233afe9225314bddd051f0ac; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60127326, jitterRate=-0.10403302311897278}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:40:56,414 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:56,414 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 8e27db14233afe9225314bddd051f0ac: Running coprocessor pre-open hook at 1733629256399Writing region info on filesystem at 1733629256399Initializing all the Stores at 1733629256400 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629256400Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629256400Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629256400Cleaning up temporary data from old regions at 1733629256410 (+10 ms)Running coprocessor post-open hooks at 1733629256414 (+4 ms)Region opened successfully at 1733629256414 2024-12-08T03:40:56,415 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., pid=57, masterSystemTime=1733629256395 2024-12-08T03:40:56,416 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,416 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:56,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=8e27db14233afe9225314bddd051f0ac, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:56,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:40:56,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-08T03:40:56,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 in 177 msec 2024-12-08T03:40:56,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-08T03:40:56,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, ASSIGN in 334 msec 2024-12-08T03:40:56,422 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:40:56,423 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629256422"}]},"ts":"1733629256422"} 2024-12-08T03:40:56,424 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:40:56,424 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:40:56,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-12-08T03:40:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-08T03:40:57,400 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:57,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5132f8a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456abfc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,412 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,415 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:40:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:40:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f7db9b8 2024-12-08T03:40:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:40:57,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:40:57,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e423066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,421 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6580a181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,423 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd3d4c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,425 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e6d5a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48def1aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545e3e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,431 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e70e1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@833877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,435 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310acfff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,437 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8a6c41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,439 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20be8f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ce1eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,443 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfc965f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,444 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,446 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2cff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd80f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,449 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29fe3376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,451 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7117e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:40:57,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:40:57,452 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:40:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:40:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:40:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b77971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:40:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:40:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:40:57,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:40:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f145e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:40:57,455 DEBUG [Thread-1098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,455 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,455 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,455 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:57,456 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,456 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,456 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,456 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,458 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,460 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:40:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,462 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:57,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,464 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:40:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:40:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:40:57,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:40:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:40:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:57,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:40:57,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629317488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629317489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629317490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629317491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629317493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/f99d5ab0d08d49df9efacfb1fd24a190 is 50, key is test_row_0/A:col10/1733629257467/Put/seqid=0 2024-12-08T03:40:57,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742058_1234 (size=14341) 2024-12-08T03:40:57,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/f99d5ab0d08d49df9efacfb1fd24a190 2024-12-08T03:40:57,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/7f5489505346439b9afa1fec071bdaf8 is 50, key is test_row_0/B:col10/1733629257467/Put/seqid=0 2024-12-08T03:40:57,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742059_1235 (size=12001) 2024-12-08T03:40:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:40:57,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629317598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629317598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629317598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629317599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629317599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-08T03:40:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:40:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:57,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-08T03:40:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:40:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:57,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:40:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629317808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629317808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629317808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629317808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629317808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:57,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-08T03:40:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:40:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:57,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:57,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/7f5489505346439b9afa1fec071bdaf8 2024-12-08T03:40:57,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/bba714844bab42e89d95154e3ec74c9c is 50, key is test_row_0/C:col10/1733629257467/Put/seqid=0 2024-12-08T03:40:57,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742060_1236 (size=12001) 2024-12-08T03:40:57,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/bba714844bab42e89d95154e3ec74c9c 2024-12-08T03:40:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/f99d5ab0d08d49df9efacfb1fd24a190 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f99d5ab0d08d49df9efacfb1fd24a190 2024-12-08T03:40:58,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f99d5ab0d08d49df9efacfb1fd24a190, entries=200, sequenceid=13, filesize=14.0 K 2024-12-08T03:40:58,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/7f5489505346439b9afa1fec071bdaf8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7f5489505346439b9afa1fec071bdaf8 2024-12-08T03:40:58,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7f5489505346439b9afa1fec071bdaf8, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T03:40:58,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/bba714844bab42e89d95154e3ec74c9c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/bba714844bab42e89d95154e3ec74c9c 2024-12-08T03:40:58,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/bba714844bab42e89d95154e3ec74c9c, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T03:40:58,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8e27db14233afe9225314bddd051f0ac in 560ms, sequenceid=13, compaction requested=false 2024-12-08T03:40:58,030 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:40:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:40:58,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-08T03:40:58,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:58,075 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:40:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/3c7fda20898c49d5a09d6a700b2a6537 is 50, key is test_row_0/A:col10/1733629257491/Put/seqid=0 2024-12-08T03:40:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:40:58,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742061_1237 (size=12001) 2024-12-08T03:40:58,100 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/3c7fda20898c49d5a09d6a700b2a6537 2024-12-08T03:40:58,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/4f404881e6f44f2e9d9f3541a24e4c4d is 50, key is test_row_0/B:col10/1733629257491/Put/seqid=0 2024-12-08T03:40:58,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742062_1238 (size=12001) 2024-12-08T03:40:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:58,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:40:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629318122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629318124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629318126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629318127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629318128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629318238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629318239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629318239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629318239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629318239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629318449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629318449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629318449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629318449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629318449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,516 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/4f404881e6f44f2e9d9f3541a24e4c4d 2024-12-08T03:40:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/d564a7f7f5e8415db876a4f74405e73a is 50, key is test_row_0/C:col10/1733629257491/Put/seqid=0 2024-12-08T03:40:58,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742063_1239 (size=12001) 2024-12-08T03:40:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:40:58,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629318758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629318759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629318759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629318759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629318759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:58,929 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/d564a7f7f5e8415db876a4f74405e73a 2024-12-08T03:40:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/3c7fda20898c49d5a09d6a700b2a6537 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/3c7fda20898c49d5a09d6a700b2a6537 2024-12-08T03:40:58,937 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/3c7fda20898c49d5a09d6a700b2a6537, entries=150, sequenceid=37, filesize=11.7 K 2024-12-08T03:40:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/4f404881e6f44f2e9d9f3541a24e4c4d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/4f404881e6f44f2e9d9f3541a24e4c4d 2024-12-08T03:40:58,941 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/4f404881e6f44f2e9d9f3541a24e4c4d, entries=150, sequenceid=37, filesize=11.7 K 2024-12-08T03:40:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/d564a7f7f5e8415db876a4f74405e73a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d564a7f7f5e8415db876a4f74405e73a 2024-12-08T03:40:58,945 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d564a7f7f5e8415db876a4f74405e73a, entries=150, sequenceid=37, filesize=11.7 K 2024-12-08T03:40:58,946 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8e27db14233afe9225314bddd051f0ac in 872ms, sequenceid=37, compaction requested=false 2024-12-08T03:40:58,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:40:58,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:58,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-08T03:40:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-08T03:40:58,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-08T03:40:58,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-12-08T03:40:58,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4930 sec 2024-12-08T03:40:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:40:59,270 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:40:59,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:40:59,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/4fe653a65d1d4920b55d88e287d0ec3b is 50, key is test_row_0/A:col10/1733629258125/Put/seqid=0 2024-12-08T03:40:59,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742064_1240 (size=16681) 2024-12-08T03:40:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629319288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629319288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629319289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629319292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629319292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,366 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:40:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629319398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629319398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629319398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629319398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629319399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:40:59,608 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:40:59,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:40:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-08T03:40:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T03:40:59,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:40:59,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629319608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629319608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,611 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:40:59,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629319608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629319609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629319609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/4fe653a65d1d4920b55d88e287d0ec3b 2024-12-08T03:40:59,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/ea511475f46f464888ed7ff42adff894 is 50, key is test_row_0/B:col10/1733629258125/Put/seqid=0 2024-12-08T03:40:59,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742065_1241 (size=12001) 2024-12-08T03:40:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T03:40:59,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-08T03:40:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:40:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:59,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:59,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-08T03:40:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:40:59,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:40:59,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:59,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:40:59,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629319918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629319918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629319918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629319918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629319918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:40:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T03:41:00,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-08T03:41:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/ea511475f46f464888ed7ff42adff894 2024-12-08T03:41:00,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/f02c707fece74626b220b5ddd8242805 is 50, key is test_row_0/C:col10/1733629258125/Put/seqid=0 2024-12-08T03:41:00,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742066_1242 (size=12001) 2024-12-08T03:41:00,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-08T03:41:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T03:41:00,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-08T03:41:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:00,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629320428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629320428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629320429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629320429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629320429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:00,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/f02c707fece74626b220b5ddd8242805 2024-12-08T03:41:00,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/4fe653a65d1d4920b55d88e287d0ec3b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/4fe653a65d1d4920b55d88e287d0ec3b 2024-12-08T03:41:00,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/4fe653a65d1d4920b55d88e287d0ec3b, entries=250, sequenceid=52, filesize=16.3 K 2024-12-08T03:41:00,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/ea511475f46f464888ed7ff42adff894 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ea511475f46f464888ed7ff42adff894 2024-12-08T03:41:00,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ea511475f46f464888ed7ff42adff894, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T03:41:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/f02c707fece74626b220b5ddd8242805 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/f02c707fece74626b220b5ddd8242805 2024-12-08T03:41:00,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/f02c707fece74626b220b5ddd8242805, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T03:41:00,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 8e27db14233afe9225314bddd051f0ac in 1251ms, sequenceid=52, compaction requested=true 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:00,521 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:00,521 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:00,522 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:00,522 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:00,522 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,523 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7f5489505346439b9afa1fec071bdaf8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/4f404881e6f44f2e9d9f3541a24e4c4d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ea511475f46f464888ed7ff42adff894] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.2 K 2024-12-08T03:41:00,523 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:00,523 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:00,523 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,523 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f5489505346439b9afa1fec071bdaf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733629257467 2024-12-08T03:41:00,523 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f99d5ab0d08d49df9efacfb1fd24a190, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/3c7fda20898c49d5a09d6a700b2a6537, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/4fe653a65d1d4920b55d88e287d0ec3b] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=42.0 K 2024-12-08T03:41:00,523 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f404881e6f44f2e9d9f3541a24e4c4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733629257488 2024-12-08T03:41:00,523 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f99d5ab0d08d49df9efacfb1fd24a190, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733629257466 2024-12-08T03:41:00,523 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ea511475f46f464888ed7ff42adff894, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733629258125 2024-12-08T03:41:00,524 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c7fda20898c49d5a09d6a700b2a6537, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733629257488 2024-12-08T03:41:00,524 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe653a65d1d4920b55d88e287d0ec3b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733629258125 2024-12-08T03:41:00,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-08T03:41:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,527 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-08T03:41:00,531 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:00,531 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:00,532 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/dcc340c659914041a272bcc3359a94cb is 50, key is test_row_0/B:col10/1733629258125/Put/seqid=0 2024-12-08T03:41:00,532 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/0ff4c80f749d451bbd178c0fb55d27a2 is 50, key is test_row_0/A:col10/1733629258125/Put/seqid=0 2024-12-08T03:41:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/969dad91d9ce404b8c6158fc445b3d1a is 50, key is test_row_0/A:col10/1733629259291/Put/seqid=0 2024-12-08T03:41:00,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742068_1244 (size=12104) 2024-12-08T03:41:00,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742067_1243 (size=12104) 2024-12-08T03:41:00,542 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/0ff4c80f749d451bbd178c0fb55d27a2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/0ff4c80f749d451bbd178c0fb55d27a2 2024-12-08T03:41:00,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742069_1245 (size=12001) 2024-12-08T03:41:00,546 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/969dad91d9ce404b8c6158fc445b3d1a 2024-12-08T03:41:00,549 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/dcc340c659914041a272bcc3359a94cb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/dcc340c659914041a272bcc3359a94cb 2024-12-08T03:41:00,552 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into 0ff4c80f749d451bbd178c0fb55d27a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:00,552 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:00,552 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=13, startTime=1733629260521; duration=0sec 2024-12-08T03:41:00,552 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:00,552 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:00,552 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:00,554 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:00,554 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:00,554 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:00,555 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/bba714844bab42e89d95154e3ec74c9c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d564a7f7f5e8415db876a4f74405e73a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/f02c707fece74626b220b5ddd8242805] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.2 K 2024-12-08T03:41:00,555 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting bba714844bab42e89d95154e3ec74c9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733629257467 2024-12-08T03:41:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/69a1ac8b04d3404b840519d97de0b14b is 50, key is test_row_0/B:col10/1733629259291/Put/seqid=0 2024-12-08T03:41:00,556 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d564a7f7f5e8415db876a4f74405e73a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733629257488 2024-12-08T03:41:00,557 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02c707fece74626b220b5ddd8242805, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733629258125 2024-12-08T03:41:00,559 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into dcc340c659914041a272bcc3359a94cb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:00,559 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:00,559 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=13, startTime=1733629260521; duration=0sec 2024-12-08T03:41:00,559 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:00,559 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742070_1246 (size=12001) 2024-12-08T03:41:00,568 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:00,568 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/dfee553dc67047fda2f78b0ebf6914a1 is 50, key is test_row_0/C:col10/1733629258125/Put/seqid=0 2024-12-08T03:41:00,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742071_1247 (size=12104) 2024-12-08T03:41:00,581 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/dfee553dc67047fda2f78b0ebf6914a1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/dfee553dc67047fda2f78b0ebf6914a1 2024-12-08T03:41:00,587 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into dfee553dc67047fda2f78b0ebf6914a1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:00,588 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:00,588 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=13, startTime=1733629260521; duration=0sec 2024-12-08T03:41:00,588 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:00,588 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T03:41:00,963 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/69a1ac8b04d3404b840519d97de0b14b 2024-12-08T03:41:00,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/04f0b6e95af44e68a968faa8bb8eb648 is 50, key is test_row_0/C:col10/1733629259291/Put/seqid=0 2024-12-08T03:41:01,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742072_1248 (size=12001) 2024-12-08T03:41:01,006 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/04f0b6e95af44e68a968faa8bb8eb648 2024-12-08T03:41:01,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/969dad91d9ce404b8c6158fc445b3d1a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/969dad91d9ce404b8c6158fc445b3d1a 2024-12-08T03:41:01,015 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/969dad91d9ce404b8c6158fc445b3d1a, entries=150, sequenceid=74, filesize=11.7 K 2024-12-08T03:41:01,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/69a1ac8b04d3404b840519d97de0b14b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/69a1ac8b04d3404b840519d97de0b14b 2024-12-08T03:41:01,020 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/69a1ac8b04d3404b840519d97de0b14b, entries=150, sequenceid=74, filesize=11.7 K 2024-12-08T03:41:01,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/04f0b6e95af44e68a968faa8bb8eb648 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04f0b6e95af44e68a968faa8bb8eb648 2024-12-08T03:41:01,025 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04f0b6e95af44e68a968faa8bb8eb648, entries=150, sequenceid=74, filesize=11.7 K 2024-12-08T03:41:01,026 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 498ms, sequenceid=74, compaction requested=false 2024-12-08T03:41:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-08T03:41:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-08T03:41:01,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-08T03:41:01,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-12-08T03:41:01,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.4200 sec 2024-12-08T03:41:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:01,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:01,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/83a51ac2abe048c0a229976861427fcd is 50, key is test_row_0/A:col10/1733629261439/Put/seqid=0 2024-12-08T03:41:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742073_1249 (size=12001) 2024-12-08T03:41:01,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/83a51ac2abe048c0a229976861427fcd 2024-12-08T03:41:01,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/964305c622394b7c8fcd569bdcd38f10 is 50, key is test_row_0/B:col10/1733629261439/Put/seqid=0 2024-12-08T03:41:01,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742074_1250 (size=12001) 2024-12-08T03:41:01,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629321469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629321474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629321475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629321475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629321476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629321578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629321588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629321588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629321588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629321588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T03:41:01,759 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:01,759 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-08T03:41:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:41:01,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:01,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:01,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629321788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629321798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629321798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629321798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629321798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:41:01,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/964305c622394b7c8fcd569bdcd38f10 2024-12-08T03:41:01,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/271938acab444521b8fb73bef66e93af is 50, key is test_row_0/C:col10/1733629261439/Put/seqid=0 2024-12-08T03:41:01,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742075_1251 (size=12001) 2024-12-08T03:41:01,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T03:41:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:01,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T03:41:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:41:02,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629322098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T03:41:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:02,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:02,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/271938acab444521b8fb73bef66e93af 2024-12-08T03:41:02,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/83a51ac2abe048c0a229976861427fcd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/83a51ac2abe048c0a229976861427fcd 2024-12-08T03:41:02,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/83a51ac2abe048c0a229976861427fcd, entries=150, sequenceid=89, filesize=11.7 K 2024-12-08T03:41:02,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/964305c622394b7c8fcd569bdcd38f10 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/964305c622394b7c8fcd569bdcd38f10 2024-12-08T03:41:02,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/964305c622394b7c8fcd569bdcd38f10, entries=150, sequenceid=89, filesize=11.7 K 2024-12-08T03:41:02,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/271938acab444521b8fb73bef66e93af as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/271938acab444521b8fb73bef66e93af 2024-12-08T03:41:02,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/271938acab444521b8fb73bef66e93af, entries=150, sequenceid=89, filesize=11.7 K 2024-12-08T03:41:02,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 8e27db14233afe9225314bddd051f0ac in 857ms, sequenceid=89, compaction requested=true 2024-12-08T03:41:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:02,304 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:02,304 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:02,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:02,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:02,305 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:02,305 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:02,305 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:02,305 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:02,306 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,306 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,306 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/0ff4c80f749d451bbd178c0fb55d27a2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/969dad91d9ce404b8c6158fc445b3d1a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/83a51ac2abe048c0a229976861427fcd] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.3 K 2024-12-08T03:41:02,306 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/dcc340c659914041a272bcc3359a94cb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/69a1ac8b04d3404b840519d97de0b14b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/964305c622394b7c8fcd569bdcd38f10] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.3 K 2024-12-08T03:41:02,306 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc340c659914041a272bcc3359a94cb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733629258125 2024-12-08T03:41:02,306 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff4c80f749d451bbd178c0fb55d27a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733629258125 2024-12-08T03:41:02,306 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a1ac8b04d3404b840519d97de0b14b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733629259279 2024-12-08T03:41:02,306 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969dad91d9ce404b8c6158fc445b3d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733629259279 2024-12-08T03:41:02,306 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 964305c622394b7c8fcd569bdcd38f10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733629261439 2024-12-08T03:41:02,306 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a51ac2abe048c0a229976861427fcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733629261439 2024-12-08T03:41:02,313 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:02,313 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:02,313 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/04f5ca29dc7940e8b86928c9b17a7a61 is 50, key is test_row_0/A:col10/1733629261439/Put/seqid=0 2024-12-08T03:41:02,313 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/0dd288dbc36b4d74b36af20e7f382ef7 is 50, key is test_row_0/B:col10/1733629261439/Put/seqid=0 2024-12-08T03:41:02,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742076_1252 (size=12207) 2024-12-08T03:41:02,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742077_1253 (size=12207) 2024-12-08T03:41:02,327 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/04f5ca29dc7940e8b86928c9b17a7a61 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/04f5ca29dc7940e8b86928c9b17a7a61 2024-12-08T03:41:02,332 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into 04f5ca29dc7940e8b86928c9b17a7a61(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:02,332 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:02,332 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=13, startTime=1733629262304; duration=0sec 2024-12-08T03:41:02,332 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:02,332 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:02,332 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:02,333 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:02,333 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:02,333 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,333 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/dfee553dc67047fda2f78b0ebf6914a1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04f0b6e95af44e68a968faa8bb8eb648, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/271938acab444521b8fb73bef66e93af] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.3 K 2024-12-08T03:41:02,333 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfee553dc67047fda2f78b0ebf6914a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733629258125 2024-12-08T03:41:02,334 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f0b6e95af44e68a968faa8bb8eb648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733629259279 2024-12-08T03:41:02,334 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271938acab444521b8fb73bef66e93af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733629261439 2024-12-08T03:41:02,340 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:02,340 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/c7e19aef83df4bd2bf5adbb04ee13fd9 is 50, key is test_row_0/C:col10/1733629261439/Put/seqid=0 2024-12-08T03:41:02,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742078_1254 (size=12207) 2024-12-08T03:41:02,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T03:41:02,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:02,371 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:41:02,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/b227fd05be5b42f9b7ba7b60bef445c8 is 50, key is test_row_0/A:col10/1733629261474/Put/seqid=0 2024-12-08T03:41:02,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742079_1255 (size=12001) 2024-12-08T03:41:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:41:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:02,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629322620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629322621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629322621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629322622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629322622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,727 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/0dd288dbc36b4d74b36af20e7f382ef7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0dd288dbc36b4d74b36af20e7f382ef7 2024-12-08T03:41:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629322728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629322728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629322728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629322728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629322728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,732 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into 0dd288dbc36b4d74b36af20e7f382ef7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,732 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,732 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=13, startTime=1733629262304; duration=0sec 2024-12-08T03:41:02,732 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:02,732 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,749 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/c7e19aef83df4bd2bf5adbb04ee13fd9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/c7e19aef83df4bd2bf5adbb04ee13fd9 2024-12-08T03:41:02,754 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into c7e19aef83df4bd2bf5adbb04ee13fd9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:02,754 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:02,754 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=13, startTime=1733629262304; duration=0sec 2024-12-08T03:41:02,754 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:02,754 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:02,780 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/b227fd05be5b42f9b7ba7b60bef445c8 2024-12-08T03:41:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/491b06e1d0854717839578ab6a8af2a7 is 50, key is test_row_0/B:col10/1733629261474/Put/seqid=0 2024-12-08T03:41:02,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742080_1256 (size=12001) 2024-12-08T03:41:02,797 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/491b06e1d0854717839578ab6a8af2a7 2024-12-08T03:41:02,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/1688fb0e12d448b6a15101551b309451 is 50, key is test_row_0/C:col10/1733629261474/Put/seqid=0 2024-12-08T03:41:02,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742081_1257 (size=12001) 2024-12-08T03:41:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:41:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,213 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/1688fb0e12d448b6a15101551b309451 2024-12-08T03:41:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/b227fd05be5b42f9b7ba7b60bef445c8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/b227fd05be5b42f9b7ba7b60bef445c8 2024-12-08T03:41:03,221 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/b227fd05be5b42f9b7ba7b60bef445c8, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T03:41:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/491b06e1d0854717839578ab6a8af2a7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/491b06e1d0854717839578ab6a8af2a7 2024-12-08T03:41:03,226 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/491b06e1d0854717839578ab6a8af2a7, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T03:41:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/1688fb0e12d448b6a15101551b309451 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/1688fb0e12d448b6a15101551b309451 2024-12-08T03:41:03,232 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/1688fb0e12d448b6a15101551b309451, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T03:41:03,232 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 8e27db14233afe9225314bddd051f0ac in 861ms, sequenceid=116, compaction requested=false 2024-12-08T03:41:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-08T03:41:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-08T03:41:03,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-08T03:41:03,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-12-08T03:41:03,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.4770 sec 2024-12-08T03:41:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:03,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:41:03,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/844d3cfa261d416f90fc67d18329705a is 50, key is test_row_0/A:col10/1733629262613/Put/seqid=0 2024-12-08T03:41:03,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742082_1258 (size=12051) 2024-12-08T03:41:03,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629323265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629323266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629323267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629323268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629323269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629323378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629323378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629323378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629323378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629323378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629323588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629323588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629323588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629323588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629323588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/844d3cfa261d416f90fc67d18329705a 2024-12-08T03:41:03,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/db5644b6193249bd865b4152919ef004 is 50, key is test_row_0/B:col10/1733629262613/Put/seqid=0 2024-12-08T03:41:03,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742083_1259 (size=12051) 2024-12-08T03:41:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:41:03,909 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:03,909 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-08T03:41:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T03:41:03,911 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:03,911 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:03,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T03:41:04,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T03:41:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:04,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/db5644b6193249bd865b4152919ef004 2024-12-08T03:41:04,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/04949aca2af84aec858f142c619fa4bc is 50, key is test_row_0/C:col10/1733629262613/Put/seqid=0 2024-12-08T03:41:04,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742084_1260 (size=12051) 2024-12-08T03:41:04,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T03:41:04,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:04,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T03:41:04,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T03:41:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629324408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629324408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629324408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629324408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629324409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:04,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/04949aca2af84aec858f142c619fa4bc 2024-12-08T03:41:04,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/844d3cfa261d416f90fc67d18329705a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/844d3cfa261d416f90fc67d18329705a 2024-12-08T03:41:04,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/844d3cfa261d416f90fc67d18329705a, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T03:41:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/db5644b6193249bd865b4152919ef004 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/db5644b6193249bd865b4152919ef004 2024-12-08T03:41:04,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/db5644b6193249bd865b4152919ef004, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T03:41:04,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/04949aca2af84aec858f142c619fa4bc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04949aca2af84aec858f142c619fa4bc 2024-12-08T03:41:04,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04949aca2af84aec858f142c619fa4bc, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T03:41:04,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8e27db14233afe9225314bddd051f0ac in 1259ms, sequenceid=131, compaction requested=true 2024-12-08T03:41:04,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:04,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:04,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:04,509 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:04,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:04,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:04,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:04,510 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:04,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:04,510 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:04,511 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,511 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,511 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0dd288dbc36b4d74b36af20e7f382ef7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/491b06e1d0854717839578ab6a8af2a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/db5644b6193249bd865b4152919ef004] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.4 K 2024-12-08T03:41:04,511 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/04f5ca29dc7940e8b86928c9b17a7a61, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/b227fd05be5b42f9b7ba7b60bef445c8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/844d3cfa261d416f90fc67d18329705a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.4 K 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f5ca29dc7940e8b86928c9b17a7a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733629261439 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd288dbc36b4d74b36af20e7f382ef7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733629261439 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 491b06e1d0854717839578ab6a8af2a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733629261473 2024-12-08T03:41:04,511 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b227fd05be5b42f9b7ba7b60bef445c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733629261473 2024-12-08T03:41:04,512 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844d3cfa261d416f90fc67d18329705a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733629262613 2024-12-08T03:41:04,512 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting db5644b6193249bd865b4152919ef004, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733629262613 2024-12-08T03:41:04,519 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:04,519 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/938a6f1d7b8a49a1bd8f0657b5c968d0 is 50, key is test_row_0/A:col10/1733629262613/Put/seqid=0 2024-12-08T03:41:04,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T03:41:04,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,522 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:41:04,532 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:04,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/38d1910347b74344bfad64f5aee3f665 is 50, key is test_row_0/A:col10/1733629263263/Put/seqid=0 2024-12-08T03:41:04,533 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/0bac04ecbcb4490d887f61b04735a897 is 50, key is test_row_0/B:col10/1733629262613/Put/seqid=0 2024-12-08T03:41:04,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742085_1261 (size=12359) 2024-12-08T03:41:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T03:41:04,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742086_1262 (size=12151) 2024-12-08T03:41:04,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742087_1263 (size=12359) 2024-12-08T03:41:04,940 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/938a6f1d7b8a49a1bd8f0657b5c968d0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/938a6f1d7b8a49a1bd8f0657b5c968d0 2024-12-08T03:41:04,941 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/38d1910347b74344bfad64f5aee3f665 2024-12-08T03:41:04,946 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into 938a6f1d7b8a49a1bd8f0657b5c968d0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:04,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:04,946 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=13, startTime=1733629264509; duration=0sec 2024-12-08T03:41:04,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:04,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:04,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:04,947 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/0bac04ecbcb4490d887f61b04735a897 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0bac04ecbcb4490d887f61b04735a897 2024-12-08T03:41:04,947 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:04,947 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:04,947 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:04,947 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/c7e19aef83df4bd2bf5adbb04ee13fd9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/1688fb0e12d448b6a15101551b309451, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04949aca2af84aec858f142c619fa4bc] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.4 K 2024-12-08T03:41:04,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/d76fe9eb8f19434ba0f041cedabb6d03 is 50, key is test_row_0/B:col10/1733629263263/Put/seqid=0 2024-12-08T03:41:04,948 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e19aef83df4bd2bf5adbb04ee13fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733629261439 2024-12-08T03:41:04,948 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1688fb0e12d448b6a15101551b309451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733629261473 2024-12-08T03:41:04,949 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04949aca2af84aec858f142c619fa4bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733629262613 2024-12-08T03:41:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742088_1264 (size=12151) 2024-12-08T03:41:04,957 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/d76fe9eb8f19434ba0f041cedabb6d03 2024-12-08T03:41:04,960 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into 0bac04ecbcb4490d887f61b04735a897(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:04,960 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:04,960 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=13, startTime=1733629264509; duration=0sec 2024-12-08T03:41:04,960 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:04,960 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:04,964 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#223 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:04,965 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/d97ed878fb6a4a10a84930b1a84d95b0 is 50, key is test_row_0/C:col10/1733629262613/Put/seqid=0 2024-12-08T03:41:04,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/80d36d924d6f4fc2b37546e36217d5a5 is 50, key is test_row_0/C:col10/1733629263263/Put/seqid=0 2024-12-08T03:41:04,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742089_1265 (size=12359) 2024-12-08T03:41:04,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742090_1266 (size=12151) 2024-12-08T03:41:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T03:41:05,385 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/d97ed878fb6a4a10a84930b1a84d95b0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d97ed878fb6a4a10a84930b1a84d95b0 2024-12-08T03:41:05,390 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/80d36d924d6f4fc2b37546e36217d5a5 2024-12-08T03:41:05,390 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into d97ed878fb6a4a10a84930b1a84d95b0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:05,391 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:05,391 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=13, startTime=1733629264510; duration=0sec 2024-12-08T03:41:05,391 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:05,391 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/38d1910347b74344bfad64f5aee3f665 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/38d1910347b74344bfad64f5aee3f665 2024-12-08T03:41:05,398 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/38d1910347b74344bfad64f5aee3f665, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T03:41:05,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/d76fe9eb8f19434ba0f041cedabb6d03 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/d76fe9eb8f19434ba0f041cedabb6d03 2024-12-08T03:41:05,403 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/d76fe9eb8f19434ba0f041cedabb6d03, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T03:41:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/80d36d924d6f4fc2b37546e36217d5a5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/80d36d924d6f4fc2b37546e36217d5a5 2024-12-08T03:41:05,409 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/80d36d924d6f4fc2b37546e36217d5a5, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T03:41:05,411 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 889ms, sequenceid=154, compaction requested=false 2024-12-08T03:41:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-08T03:41:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-08T03:41:05,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-08T03:41:05,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-12-08T03:41:05,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5060 sec 2024-12-08T03:41:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:05,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:05,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/94b394a3c7f248fcad868720fd8bf794 is 50, key is test_row_0/A:col10/1733629265423/Put/seqid=0 2024-12-08T03:41:05,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742091_1267 (size=16931) 2024-12-08T03:41:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629325442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629325444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629325445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629325446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629325447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629325558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629325559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629325559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629325559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629325559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629325769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629325769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629325769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629325769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629325769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:05,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/94b394a3c7f248fcad868720fd8bf794 2024-12-08T03:41:05,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/1861590dc7d44b9abb8b2d7d58f9b032 is 50, key is test_row_0/B:col10/1733629265423/Put/seqid=0 2024-12-08T03:41:05,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742092_1268 (size=12151) 2024-12-08T03:41:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T03:41:06,068 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:06,069 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-08T03:41:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:41:06,070 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:06,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:06,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:06,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629326078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629326078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629326078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629326079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629326079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:41:06,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:41:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/1861590dc7d44b9abb8b2d7d58f9b032 2024-12-08T03:41:06,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/379e1ecbfdf74046874096634963eb3f is 50, key is test_row_0/C:col10/1733629265423/Put/seqid=0 2024-12-08T03:41:06,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742093_1269 (size=12151) 2024-12-08T03:41:06,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:41:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:41:06,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:41:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:06,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629326588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629326588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629326588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629326588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629326588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:06,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/379e1ecbfdf74046874096634963eb3f 2024-12-08T03:41:06,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/94b394a3c7f248fcad868720fd8bf794 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94b394a3c7f248fcad868720fd8bf794 2024-12-08T03:41:06,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94b394a3c7f248fcad868720fd8bf794, entries=250, sequenceid=168, filesize=16.5 K 2024-12-08T03:41:06,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/1861590dc7d44b9abb8b2d7d58f9b032 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/1861590dc7d44b9abb8b2d7d58f9b032 2024-12-08T03:41:06,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/1861590dc7d44b9abb8b2d7d58f9b032, entries=150, sequenceid=168, filesize=11.9 K 2024-12-08T03:41:06,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/379e1ecbfdf74046874096634963eb3f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/379e1ecbfdf74046874096634963eb3f 2024-12-08T03:41:06,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/379e1ecbfdf74046874096634963eb3f, entries=150, sequenceid=168, filesize=11.9 K 2024-12-08T03:41:06,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8e27db14233afe9225314bddd051f0ac in 1250ms, sequenceid=168, compaction requested=true 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:06,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:06,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:06,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:06,676 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,676 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,676 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/938a6f1d7b8a49a1bd8f0657b5c968d0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/38d1910347b74344bfad64f5aee3f665, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94b394a3c7f248fcad868720fd8bf794] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=40.5 K 2024-12-08T03:41:06,676 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0bac04ecbcb4490d887f61b04735a897, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/d76fe9eb8f19434ba0f041cedabb6d03, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/1861590dc7d44b9abb8b2d7d58f9b032] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.8 K 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938a6f1d7b8a49a1bd8f0657b5c968d0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733629262613 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bac04ecbcb4490d887f61b04735a897, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733629262613 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38d1910347b74344bfad64f5aee3f665, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629263263 2024-12-08T03:41:06,676 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d76fe9eb8f19434ba0f041cedabb6d03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629263263 2024-12-08T03:41:06,677 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b394a3c7f248fcad868720fd8bf794, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629265420 2024-12-08T03:41:06,677 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1861590dc7d44b9abb8b2d7d58f9b032, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629265421 2024-12-08T03:41:06,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:41:06,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:06,681 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:41:06,684 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:06,684 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:06,685 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/ac00ffb077724378afee25b6628ec5f0 is 50, key is test_row_0/A:col10/1733629265423/Put/seqid=0 2024-12-08T03:41:06,685 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/c890568024254f19aab0d5a50d576d9b is 50, key is test_row_0/B:col10/1733629265423/Put/seqid=0 2024-12-08T03:41:06,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/94dd451d70504c72822113d47e556d52 is 50, key is test_row_0/A:col10/1733629265445/Put/seqid=0 2024-12-08T03:41:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:41:06,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742095_1271 (size=12561) 2024-12-08T03:41:06,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742094_1270 (size=12561) 2024-12-08T03:41:06,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742096_1272 (size=12151) 2024-12-08T03:41:06,737 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/94dd451d70504c72822113d47e556d52 2024-12-08T03:41:06,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/cfbe6f4ea42d40fa9df9eb2942432e66 is 50, key is test_row_0/B:col10/1733629265445/Put/seqid=0 2024-12-08T03:41:06,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742097_1273 (size=12151) 2024-12-08T03:41:06,752 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/cfbe6f4ea42d40fa9df9eb2942432e66 2024-12-08T03:41:06,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/ff1584dcb8c840ecb675042ac7683580 is 50, key is test_row_0/C:col10/1733629265445/Put/seqid=0 2024-12-08T03:41:06,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742098_1274 (size=12151) 2024-12-08T03:41:07,156 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/ac00ffb077724378afee25b6628ec5f0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/ac00ffb077724378afee25b6628ec5f0 2024-12-08T03:41:07,156 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/c890568024254f19aab0d5a50d576d9b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c890568024254f19aab0d5a50d576d9b 2024-12-08T03:41:07,162 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into c890568024254f19aab0d5a50d576d9b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:07,162 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:07,162 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into ac00ffb077724378afee25b6628ec5f0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:07,162 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=13, startTime=1733629266675; duration=0sec 2024-12-08T03:41:07,162 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:07,162 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=13, startTime=1733629266675; duration=0sec 2024-12-08T03:41:07,162 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:07,162 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:07,162 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:07,163 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:07,163 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:07,163 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:07,163 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:07,163 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,163 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d97ed878fb6a4a10a84930b1a84d95b0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/80d36d924d6f4fc2b37546e36217d5a5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/379e1ecbfdf74046874096634963eb3f] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=35.8 K 2024-12-08T03:41:07,164 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d97ed878fb6a4a10a84930b1a84d95b0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733629262613 2024-12-08T03:41:07,164 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/ff1584dcb8c840ecb675042ac7683580 2024-12-08T03:41:07,164 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d36d924d6f4fc2b37546e36217d5a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629263263 2024-12-08T03:41:07,164 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 379e1ecbfdf74046874096634963eb3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629265421 2024-12-08T03:41:07,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/94dd451d70504c72822113d47e556d52 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94dd451d70504c72822113d47e556d52 2024-12-08T03:41:07,172 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:07,172 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/cef2283bf72446a4affaedfb6286c7cb is 50, key is test_row_0/C:col10/1733629265423/Put/seqid=0 2024-12-08T03:41:07,174 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94dd451d70504c72822113d47e556d52, entries=150, sequenceid=193, filesize=11.9 K 2024-12-08T03:41:07,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/cfbe6f4ea42d40fa9df9eb2942432e66 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/cfbe6f4ea42d40fa9df9eb2942432e66 2024-12-08T03:41:07,179 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/cfbe6f4ea42d40fa9df9eb2942432e66, entries=150, sequenceid=193, filesize=11.9 K 2024-12-08T03:41:07,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/ff1584dcb8c840ecb675042ac7683580 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ff1584dcb8c840ecb675042ac7683580 2024-12-08T03:41:07,184 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ff1584dcb8c840ecb675042ac7683580, entries=150, sequenceid=193, filesize=11.9 K 2024-12-08T03:41:07,185 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 504ms, sequenceid=193, compaction requested=false 2024-12-08T03:41:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-08T03:41:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-08T03:41:07,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-08T03:41:07,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1150 sec 2024-12-08T03:41:07,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1200 sec 2024-12-08T03:41:07,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742099_1275 (size=12561) 2024-12-08T03:41:07,203 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/cef2283bf72446a4affaedfb6286c7cb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cef2283bf72446a4affaedfb6286c7cb 2024-12-08T03:41:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:41:07,209 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:07,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:07,210 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into cef2283bf72446a4affaedfb6286c7cb(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:07,210 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:07,210 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=13, startTime=1733629266675; duration=0sec 2024-12-08T03:41:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-08T03:41:07,210 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:07,210 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:41:07,211 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:07,211 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:07,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:41:07,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:41:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-08T03:41:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-08T03:41:07,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-08T03:41:07,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-08T03:41:07,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 159 msec 2024-12-08T03:41:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:41:07,528 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:07,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-08T03:41:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:41:07,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:07,531 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:07,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:07,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:07,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/e0e1f1c0e7754c5aa1d3d3136d560426 is 50, key is test_row_0/A:col10/1733629267602/Put/seqid=0 2024-12-08T03:41:07,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742100_1276 (size=16931) 2024-12-08T03:41:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629327623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629327623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629327624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629327627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629327628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:41:07,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:07,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629327738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629327738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629327738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629327739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629327739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:41:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629327948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629327948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629327949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629327949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629327949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:07,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:07,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/e0e1f1c0e7754c5aa1d3d3136d560426 2024-12-08T03:41:08,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/a836244c9605467b8e107c1fb9fe70ff is 50, key is test_row_0/B:col10/1733629267602/Put/seqid=0 2024-12-08T03:41:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742101_1277 (size=12151) 2024-12-08T03:41:08,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/a836244c9605467b8e107c1fb9fe70ff 2024-12-08T03:41:08,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/ea36d494b05a4759b668d7ccc9a8a087 is 50, key is test_row_0/C:col10/1733629267602/Put/seqid=0 2024-12-08T03:41:08,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742102_1278 (size=12151) 2024-12-08T03:41:08,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:41:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629328258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629328258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629328258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629328258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629328258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/ea36d494b05a4759b668d7ccc9a8a087 2024-12-08T03:41:08,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:08,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:08,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/e0e1f1c0e7754c5aa1d3d3136d560426 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/e0e1f1c0e7754c5aa1d3d3136d560426 2024-12-08T03:41:08,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/e0e1f1c0e7754c5aa1d3d3136d560426, entries=250, sequenceid=208, filesize=16.5 K 2024-12-08T03:41:08,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/a836244c9605467b8e107c1fb9fe70ff as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/a836244c9605467b8e107c1fb9fe70ff 2024-12-08T03:41:08,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/a836244c9605467b8e107c1fb9fe70ff, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T03:41:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/ea36d494b05a4759b668d7ccc9a8a087 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ea36d494b05a4759b668d7ccc9a8a087 2024-12-08T03:41:08,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ea36d494b05a4759b668d7ccc9a8a087, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T03:41:08,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8e27db14233afe9225314bddd051f0ac in 860ms, sequenceid=208, compaction requested=true 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:08,463 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:08,463 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:08,464 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:08,464 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:08,464 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:08,464 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:08,464 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,464 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,464 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c890568024254f19aab0d5a50d576d9b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/cfbe6f4ea42d40fa9df9eb2942432e66, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/a836244c9605467b8e107c1fb9fe70ff] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=36.0 K 2024-12-08T03:41:08,464 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/ac00ffb077724378afee25b6628ec5f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94dd451d70504c72822113d47e556d52, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/e0e1f1c0e7754c5aa1d3d3136d560426] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=40.7 K 2024-12-08T03:41:08,465 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac00ffb077724378afee25b6628ec5f0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629265421 2024-12-08T03:41:08,465 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c890568024254f19aab0d5a50d576d9b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629265421 2024-12-08T03:41:08,465 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94dd451d70504c72822113d47e556d52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733629265441 2024-12-08T03:41:08,465 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cfbe6f4ea42d40fa9df9eb2942432e66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733629265441 2024-12-08T03:41:08,465 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e1f1c0e7754c5aa1d3d3136d560426, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629267599 2024-12-08T03:41:08,465 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a836244c9605467b8e107c1fb9fe70ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629267601 2024-12-08T03:41:08,472 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:08,473 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/f616b58058e843fa81631b6e30a45bd7 is 50, key is test_row_0/A:col10/1733629267602/Put/seqid=0 2024-12-08T03:41:08,481 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#238 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:08,481 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/75fe0fd0925e4615950d1a8ae8f67c16 is 50, key is test_row_0/B:col10/1733629267602/Put/seqid=0 2024-12-08T03:41:08,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742103_1279 (size=12663) 2024-12-08T03:41:08,497 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/f616b58058e843fa81631b6e30a45bd7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f616b58058e843fa81631b6e30a45bd7 2024-12-08T03:41:08,502 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into f616b58058e843fa81631b6e30a45bd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:08,502 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:08,502 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=13, startTime=1733629268463; duration=0sec 2024-12-08T03:41:08,502 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:08,502 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:08,502 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:08,503 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:08,503 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:08,503 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,503 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cef2283bf72446a4affaedfb6286c7cb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ff1584dcb8c840ecb675042ac7683580, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ea36d494b05a4759b668d7ccc9a8a087] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=36.0 K 2024-12-08T03:41:08,504 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef2283bf72446a4affaedfb6286c7cb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629265421 2024-12-08T03:41:08,504 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1584dcb8c840ecb675042ac7683580, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733629265441 2024-12-08T03:41:08,505 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea36d494b05a4759b668d7ccc9a8a087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629267601 2024-12-08T03:41:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742104_1280 (size=12663) 2024-12-08T03:41:08,513 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:08,513 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/9ac991a5b239437e991b7bc769b9c1c1 is 50, key is test_row_0/C:col10/1733629267602/Put/seqid=0 2024-12-08T03:41:08,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742105_1281 (size=12663) 2024-12-08T03:41:08,530 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/9ac991a5b239437e991b7bc769b9c1c1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9ac991a5b239437e991b7bc769b9c1c1 2024-12-08T03:41:08,536 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into 9ac991a5b239437e991b7bc769b9c1c1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:08,536 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:08,536 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=13, startTime=1733629268463; duration=0sec 2024-12-08T03:41:08,536 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:08,536 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:08,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:41:08,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:08,599 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:41:08,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/d5ba78995d91417a83f2cd94af08bd65 is 50, key is test_row_0/A:col10/1733629267626/Put/seqid=0 2024-12-08T03:41:08,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742106_1282 (size=12151) 2024-12-08T03:41:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:41:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629328773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629328773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629328775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629328776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629328776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629328878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629328888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629328888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629328888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629328888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:08,910 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/75fe0fd0925e4615950d1a8ae8f67c16 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/75fe0fd0925e4615950d1a8ae8f67c16 2024-12-08T03:41:08,916 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into 75fe0fd0925e4615950d1a8ae8f67c16(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:08,916 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:08,916 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=13, startTime=1733629268463; duration=0sec 2024-12-08T03:41:08,916 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:08,916 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:09,008 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/d5ba78995d91417a83f2cd94af08bd65 2024-12-08T03:41:09,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/03902c818cc146d2be339b7ec25cc87e is 50, key is test_row_0/B:col10/1733629267626/Put/seqid=0 2024-12-08T03:41:09,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742107_1283 (size=12151) 2024-12-08T03:41:09,021 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/03902c818cc146d2be339b7ec25cc87e 2024-12-08T03:41:09,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/452244b916cf467c9b4dc21f5dfde7da is 50, key is test_row_0/C:col10/1733629267626/Put/seqid=0 2024-12-08T03:41:09,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742108_1284 (size=12151) 2024-12-08T03:41:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629329088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629329399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629329408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629329408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629329408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629329408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,431 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/452244b916cf467c9b4dc21f5dfde7da 2024-12-08T03:41:09,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/d5ba78995d91417a83f2cd94af08bd65 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/d5ba78995d91417a83f2cd94af08bd65 2024-12-08T03:41:09,438 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/d5ba78995d91417a83f2cd94af08bd65, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:41:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/03902c818cc146d2be339b7ec25cc87e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/03902c818cc146d2be339b7ec25cc87e 2024-12-08T03:41:09,443 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/03902c818cc146d2be339b7ec25cc87e, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:41:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/452244b916cf467c9b4dc21f5dfde7da as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/452244b916cf467c9b4dc21f5dfde7da 2024-12-08T03:41:09,447 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/452244b916cf467c9b4dc21f5dfde7da, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:41:09,448 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8e27db14233afe9225314bddd051f0ac in 849ms, sequenceid=235, compaction requested=false 2024-12-08T03:41:09,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:09,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:09,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-08T03:41:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-08T03:41:09,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-08T03:41:09,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-12-08T03:41:09,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9240 sec 2024-12-08T03:41:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:41:09,678 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:09,679 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-08T03:41:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:41:09,682 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:09,683 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:09,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:41:09,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:41:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:09,836 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:09,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/fc5dea8d29af4f588942d48d540f69b2 is 50, key is test_row_0/A:col10/1733629268775/Put/seqid=0 2024-12-08T03:41:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742109_1285 (size=12151) 2024-12-08T03:41:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:09,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:09,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629329933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629329933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629329935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629329936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629329937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:41:10,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629330048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629330048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629330048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629330048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629330048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,244 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/fc5dea8d29af4f588942d48d540f69b2 2024-12-08T03:41:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629330259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629330259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629330259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629330259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629330259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/ef0878f67aa9471a9a12954e215648d6 is 50, key is test_row_0/B:col10/1733629268775/Put/seqid=0 2024-12-08T03:41:10,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742110_1286 (size=12151) 2024-12-08T03:41:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:41:10,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629330568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629330569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629330569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629330569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629330569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:10,678 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/ef0878f67aa9471a9a12954e215648d6 2024-12-08T03:41:10,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/8943b3490f4b44e3a6b0137afe8f04bc is 50, key is test_row_0/C:col10/1733629268775/Put/seqid=0 2024-12-08T03:41:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742111_1287 (size=12151) 2024-12-08T03:41:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:41:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629331078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629331078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629331078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629331078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629331078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:11,091 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/8943b3490f4b44e3a6b0137afe8f04bc 2024-12-08T03:41:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/fc5dea8d29af4f588942d48d540f69b2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/fc5dea8d29af4f588942d48d540f69b2 2024-12-08T03:41:11,098 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/fc5dea8d29af4f588942d48d540f69b2, entries=150, sequenceid=247, filesize=11.9 K 2024-12-08T03:41:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/ef0878f67aa9471a9a12954e215648d6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ef0878f67aa9471a9a12954e215648d6 2024-12-08T03:41:11,102 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ef0878f67aa9471a9a12954e215648d6, entries=150, sequenceid=247, filesize=11.9 K 2024-12-08T03:41:11,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/8943b3490f4b44e3a6b0137afe8f04bc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8943b3490f4b44e3a6b0137afe8f04bc 2024-12-08T03:41:11,107 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8943b3490f4b44e3a6b0137afe8f04bc, entries=150, sequenceid=247, filesize=11.9 K 2024-12-08T03:41:11,107 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8e27db14233afe9225314bddd051f0ac in 1271ms, sequenceid=247, compaction requested=true 2024-12-08T03:41:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-08T03:41:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-08T03:41:11,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-08T03:41:11,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-12-08T03:41:11,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4320 sec 2024-12-08T03:41:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:41:11,828 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:11,828 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-08T03:41:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:41:11,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:11,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:11,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:41:11,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T03:41:11,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:11,983 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:41:11,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/c1ed9bb794a84804b52d90a25fdc605f is 50, key is test_row_0/A:col10/1733629269935/Put/seqid=0 2024-12-08T03:41:11,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742112_1288 (size=12301) 2024-12-08T03:41:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:12,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629332102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629332102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629332102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629332103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629332103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:41:12,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629332208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629332208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629332209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629332218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629332218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,390 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/c1ed9bb794a84804b52d90a25fdc605f 2024-12-08T03:41:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/7506f6339c664b6a89d16f559ed91ee4 is 50, key is test_row_0/B:col10/1733629269935/Put/seqid=0 2024-12-08T03:41:12,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742113_1289 (size=12301) 2024-12-08T03:41:12,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629332418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629332418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629332418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629332428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629332428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:41:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629332728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629332728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629332728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629332738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629332739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:12,803 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/7506f6339c664b6a89d16f559ed91ee4 2024-12-08T03:41:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/74104f1a10264085bbd5d1e01ac34a8d is 50, key is test_row_0/C:col10/1733629269935/Put/seqid=0 2024-12-08T03:41:12,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742114_1290 (size=12301) 2024-12-08T03:41:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:41:13,213 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/74104f1a10264085bbd5d1e01ac34a8d 2024-12-08T03:41:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/c1ed9bb794a84804b52d90a25fdc605f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/c1ed9bb794a84804b52d90a25fdc605f 2024-12-08T03:41:13,221 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/c1ed9bb794a84804b52d90a25fdc605f, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T03:41:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/7506f6339c664b6a89d16f559ed91ee4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7506f6339c664b6a89d16f559ed91ee4 2024-12-08T03:41:13,226 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7506f6339c664b6a89d16f559ed91ee4, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T03:41:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/74104f1a10264085bbd5d1e01ac34a8d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/74104f1a10264085bbd5d1e01ac34a8d 2024-12-08T03:41:13,230 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/74104f1a10264085bbd5d1e01ac34a8d, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T03:41:13,231 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 8e27db14233afe9225314bddd051f0ac in 1248ms, sequenceid=272, compaction requested=true 2024-12-08T03:41:13,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:13,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:13,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-08T03:41:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-08T03:41:13,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-08T03:41:13,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-12-08T03:41:13,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4050 sec 2024-12-08T03:41:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:13,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:13,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:41:13,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/2411ada05c6d49daafd0d951b30d7125 is 50, key is test_row_0/A:col10/1733629272101/Put/seqid=0 2024-12-08T03:41:13,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742115_1291 (size=14741) 2024-12-08T03:41:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629333263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629333264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629333264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629333264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629333265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629333378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629333378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629333378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629333378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629333378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629333588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629333588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629333588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629333588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629333588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/2411ada05c6d49daafd0d951b30d7125 2024-12-08T03:41:13,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/c91c4bdd61fb4c4ebf688074ea5acabe is 50, key is test_row_0/B:col10/1733629272101/Put/seqid=0 2024-12-08T03:41:13,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742116_1292 (size=12301) 2024-12-08T03:41:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629333898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629333899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629333899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629333899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629333899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:41:13,978 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:13,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-08T03:41:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:41:13,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:13,981 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:13,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:14,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/c91c4bdd61fb4c4ebf688074ea5acabe 2024-12-08T03:41:14,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/8d98a7640b5e4e59beb23ebfcc977b90 is 50, key is test_row_0/C:col10/1733629272101/Put/seqid=0 2024-12-08T03:41:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742117_1293 (size=12301) 2024-12-08T03:41:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:41:14,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:41:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:41:14,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629334408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629334408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629334408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:14,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629334408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:14,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629334408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:14,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:41:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:14,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/8d98a7640b5e4e59beb23ebfcc977b90 2024-12-08T03:41:14,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/2411ada05c6d49daafd0d951b30d7125 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/2411ada05c6d49daafd0d951b30d7125 2024-12-08T03:41:14,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/2411ada05c6d49daafd0d951b30d7125, entries=200, sequenceid=285, filesize=14.4 K 2024-12-08T03:41:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/c91c4bdd61fb4c4ebf688074ea5acabe as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c91c4bdd61fb4c4ebf688074ea5acabe 2024-12-08T03:41:14,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c91c4bdd61fb4c4ebf688074ea5acabe, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T03:41:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/8d98a7640b5e4e59beb23ebfcc977b90 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8d98a7640b5e4e59beb23ebfcc977b90 2024-12-08T03:41:14,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8d98a7640b5e4e59beb23ebfcc977b90, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T03:41:14,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8e27db14233afe9225314bddd051f0ac in 1247ms, sequenceid=285, compaction requested=true 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:14,488 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:14,488 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:14,489 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64007 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T03:41:14,489 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61567 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T03:41:14,489 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:14,489 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:14,489 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,489 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,489 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f616b58058e843fa81631b6e30a45bd7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/d5ba78995d91417a83f2cd94af08bd65, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/fc5dea8d29af4f588942d48d540f69b2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/c1ed9bb794a84804b52d90a25fdc605f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/2411ada05c6d49daafd0d951b30d7125] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=62.5 K 2024-12-08T03:41:14,489 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/75fe0fd0925e4615950d1a8ae8f67c16, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/03902c818cc146d2be339b7ec25cc87e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ef0878f67aa9471a9a12954e215648d6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7506f6339c664b6a89d16f559ed91ee4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c91c4bdd61fb4c4ebf688074ea5acabe] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=60.1 K 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f616b58058e843fa81631b6e30a45bd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629267601 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 75fe0fd0925e4615950d1a8ae8f67c16, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629267601 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 03902c818cc146d2be339b7ec25cc87e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629267623 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5ba78995d91417a83f2cd94af08bd65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629267623 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0878f67aa9471a9a12954e215648d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733629268774 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc5dea8d29af4f588942d48d540f69b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733629268774 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7506f6339c664b6a89d16f559ed91ee4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733629269934 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ed9bb794a84804b52d90a25fdc605f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733629269934 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c91c4bdd61fb4c4ebf688074ea5acabe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733629272101 2024-12-08T03:41:14,490 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2411ada05c6d49daafd0d951b30d7125, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733629272101 2024-12-08T03:41:14,499 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:14,500 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/521eb065572a4bdc911e3cda5da9591d is 50, key is test_row_0/A:col10/1733629272101/Put/seqid=0 2024-12-08T03:41:14,500 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:14,500 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/e7fc7d4422694570b968af5c6f430eaa is 50, key is test_row_0/B:col10/1733629272101/Put/seqid=0 2024-12-08T03:41:14,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742119_1295 (size=12983) 2024-12-08T03:41:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742118_1294 (size=12983) 2024-12-08T03:41:14,511 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/e7fc7d4422694570b968af5c6f430eaa as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e7fc7d4422694570b968af5c6f430eaa 2024-12-08T03:41:14,516 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into e7fc7d4422694570b968af5c6f430eaa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:14,517 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:14,517 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=11, startTime=1733629274488; duration=0sec 2024-12-08T03:41:14,517 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:14,517 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:14,517 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T03:41:14,519 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61567 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T03:41:14,519 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:14,519 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,519 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9ac991a5b239437e991b7bc769b9c1c1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/452244b916cf467c9b4dc21f5dfde7da, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8943b3490f4b44e3a6b0137afe8f04bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/74104f1a10264085bbd5d1e01ac34a8d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8d98a7640b5e4e59beb23ebfcc977b90] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=60.1 K 2024-12-08T03:41:14,519 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac991a5b239437e991b7bc769b9c1c1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733629267601 2024-12-08T03:41:14,520 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 452244b916cf467c9b4dc21f5dfde7da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629267623 2024-12-08T03:41:14,520 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8943b3490f4b44e3a6b0137afe8f04bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733629268774 2024-12-08T03:41:14,520 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 74104f1a10264085bbd5d1e01ac34a8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733629269934 2024-12-08T03:41:14,520 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d98a7640b5e4e59beb23ebfcc977b90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733629272101 2024-12-08T03:41:14,529 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:14,530 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/cfccc0ef28684b48a73ac86db823f74d is 50, key is test_row_0/C:col10/1733629272101/Put/seqid=0 2024-12-08T03:41:14,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742120_1296 (size=12983) 2024-12-08T03:41:14,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:41:14,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,592 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/9149162ab89a40a4a033a8ef3f147dda is 50, key is test_row_0/A:col10/1733629273263/Put/seqid=0 2024-12-08T03:41:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742121_1297 (size=12301) 2024-12-08T03:41:14,601 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/9149162ab89a40a4a033a8ef3f147dda 2024-12-08T03:41:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:41:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/df11c6dbba394046a88a86309a4ab8eb is 50, key is test_row_0/B:col10/1733629273263/Put/seqid=0 2024-12-08T03:41:14,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742122_1298 (size=12301) 2024-12-08T03:41:14,615 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/df11c6dbba394046a88a86309a4ab8eb 2024-12-08T03:41:14,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/007f63c1e2cc45e4b04bcf30cd2ddb24 is 50, key is test_row_0/C:col10/1733629273263/Put/seqid=0 2024-12-08T03:41:14,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742123_1299 (size=12301) 2024-12-08T03:41:14,630 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/007f63c1e2cc45e4b04bcf30cd2ddb24 2024-12-08T03:41:14,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/9149162ab89a40a4a033a8ef3f147dda as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9149162ab89a40a4a033a8ef3f147dda 2024-12-08T03:41:14,644 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9149162ab89a40a4a033a8ef3f147dda, entries=150, sequenceid=309, filesize=12.0 K 2024-12-08T03:41:14,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/df11c6dbba394046a88a86309a4ab8eb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/df11c6dbba394046a88a86309a4ab8eb 2024-12-08T03:41:14,652 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/df11c6dbba394046a88a86309a4ab8eb, entries=150, sequenceid=309, filesize=12.0 K 2024-12-08T03:41:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/007f63c1e2cc45e4b04bcf30cd2ddb24 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/007f63c1e2cc45e4b04bcf30cd2ddb24 2024-12-08T03:41:14,659 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/007f63c1e2cc45e4b04bcf30cd2ddb24, entries=150, sequenceid=309, filesize=12.0 K 2024-12-08T03:41:14,660 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 68ms, sequenceid=309, compaction requested=false 2024-12-08T03:41:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-08T03:41:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-08T03:41:14,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-08T03:41:14,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 681 msec 2024-12-08T03:41:14,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 688 msec 2024-12-08T03:41:14,912 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/521eb065572a4bdc911e3cda5da9591d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/521eb065572a4bdc911e3cda5da9591d 2024-12-08T03:41:14,920 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into 521eb065572a4bdc911e3cda5da9591d(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:14,920 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:14,920 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=11, startTime=1733629274488; duration=0sec 2024-12-08T03:41:14,920 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:14,920 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:14,947 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/cfccc0ef28684b48a73ac86db823f74d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cfccc0ef28684b48a73ac86db823f74d 2024-12-08T03:41:14,953 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into cfccc0ef28684b48a73ac86db823f74d(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:14,953 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:14,953 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=11, startTime=1733629274488; duration=0sec 2024-12-08T03:41:14,953 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:14,953 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:41:15,118 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:15,118 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-08T03:41:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T03:41:15,120 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:15,120 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:15,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T03:41:15,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T03:41:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-08T03:41:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-08T03:41:15,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-08T03:41:15,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-08T03:41:15,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 158 msec 2024-12-08T03:41:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:15,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:15,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:15,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/9a59db0efadf49568b0b1578bf05252f is 50, key is test_row_0/A:col10/1733629275424/Put/seqid=0 2024-12-08T03:41:15,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742124_1300 (size=12301) 2024-12-08T03:41:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T03:41:15,439 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:15,439 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-08T03:41:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:41:15,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:15,441 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:15,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629335451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629335452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629335452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629335453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629335453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:41:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629335558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629335558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629335559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629335559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629335559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:15,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:15,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:41:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629335768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629335769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629335769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629335769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629335769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:15,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/9a59db0efadf49568b0b1578bf05252f 2024-12-08T03:41:15,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/f6299543b25a41f3b9ec19cae8df2cb3 is 50, key is test_row_0/B:col10/1733629275424/Put/seqid=0 2024-12-08T03:41:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742125_1301 (size=12301) 2024-12-08T03:41:15,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:15,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:15,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:16,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:16,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:41:16,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629336078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629336078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629336078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629336078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629336078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/f6299543b25a41f3b9ec19cae8df2cb3 2024-12-08T03:41:16,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/13ad1b89e6374e4eb76112495ef34699 is 50, key is test_row_0/C:col10/1733629275424/Put/seqid=0 2024-12-08T03:41:16,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742126_1302 (size=12301) 2024-12-08T03:41:16,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:16,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:16,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:41:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1733629336588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54508 deadline: 1733629336588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54564 deadline: 1733629336588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54548 deadline: 1733629336589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1733629336589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:16,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/13ad1b89e6374e4eb76112495ef34699 2024-12-08T03:41:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/9a59db0efadf49568b0b1578bf05252f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9a59db0efadf49568b0b1578bf05252f 2024-12-08T03:41:16,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. as already flushing 2024-12-08T03:41:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:16,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9a59db0efadf49568b0b1578bf05252f, entries=150, sequenceid=324, filesize=12.0 K 2024-12-08T03:41:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/f6299543b25a41f3b9ec19cae8df2cb3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/f6299543b25a41f3b9ec19cae8df2cb3 2024-12-08T03:41:16,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/f6299543b25a41f3b9ec19cae8df2cb3, entries=150, sequenceid=324, filesize=12.0 K 2024-12-08T03:41:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/13ad1b89e6374e4eb76112495ef34699 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/13ad1b89e6374e4eb76112495ef34699 2024-12-08T03:41:16,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/13ad1b89e6374e4eb76112495ef34699, entries=150, sequenceid=324, filesize=12.0 K 2024-12-08T03:41:16,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8e27db14233afe9225314bddd051f0ac in 1249ms, sequenceid=324, compaction requested=true 2024-12-08T03:41:16,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:16,674 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:16,674 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e27db14233afe9225314bddd051f0ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:16,674 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:16,674 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:16,674 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/B is initiating minor compaction (all files) 2024-12-08T03:41:16,674 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/A is initiating minor compaction (all files) 2024-12-08T03:41:16,675 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/B in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/A in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,675 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e7fc7d4422694570b968af5c6f430eaa, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/df11c6dbba394046a88a86309a4ab8eb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/f6299543b25a41f3b9ec19cae8df2cb3] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=36.7 K 2024-12-08T03:41:16,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/521eb065572a4bdc911e3cda5da9591d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9149162ab89a40a4a033a8ef3f147dda, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9a59db0efadf49568b0b1578bf05252f] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=36.7 K 2024-12-08T03:41:16,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting e7fc7d4422694570b968af5c6f430eaa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733629272101 2024-12-08T03:41:16,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521eb065572a4bdc911e3cda5da9591d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733629272101 2024-12-08T03:41:16,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9149162ab89a40a4a033a8ef3f147dda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733629273263 2024-12-08T03:41:16,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting df11c6dbba394046a88a86309a4ab8eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733629273263 2024-12-08T03:41:16,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a59db0efadf49568b0b1578bf05252f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733629275421 2024-12-08T03:41:16,676 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f6299543b25a41f3b9ec19cae8df2cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733629275421 2024-12-08T03:41:16,681 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:16,682 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:16,682 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/0fda8bb1192d41cd82462b80ee845344 is 50, key is test_row_0/B:col10/1733629275424/Put/seqid=0 2024-12-08T03:41:16,682 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/5c2708380cff41f68cdabc4af789ab4c is 50, key is test_row_0/A:col10/1733629275424/Put/seqid=0 2024-12-08T03:41:16,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742127_1303 (size=13085) 2024-12-08T03:41:16,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742128_1304 (size=13085) 2024-12-08T03:41:16,702 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/0fda8bb1192d41cd82462b80ee845344 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0fda8bb1192d41cd82462b80ee845344 2024-12-08T03:41:16,719 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/B of 8e27db14233afe9225314bddd051f0ac into 0fda8bb1192d41cd82462b80ee845344(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:16,720 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:16,720 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/B, priority=13, startTime=1733629276674; duration=0sec 2024-12-08T03:41:16,720 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:16,720 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:B 2024-12-08T03:41:16,720 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:16,721 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:16,722 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 8e27db14233afe9225314bddd051f0ac/C is initiating minor compaction (all files) 2024-12-08T03:41:16,722 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e27db14233afe9225314bddd051f0ac/C in TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,722 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cfccc0ef28684b48a73ac86db823f74d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/007f63c1e2cc45e4b04bcf30cd2ddb24, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/13ad1b89e6374e4eb76112495ef34699] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp, totalSize=36.7 K 2024-12-08T03:41:16,722 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cfccc0ef28684b48a73ac86db823f74d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733629272101 2024-12-08T03:41:16,723 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 007f63c1e2cc45e4b04bcf30cd2ddb24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733629273263 2024-12-08T03:41:16,723 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ad1b89e6374e4eb76112495ef34699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733629275421 2024-12-08T03:41:16,750 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e27db14233afe9225314bddd051f0ac#C#compaction#263 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:16,751 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/9d3fca55b3194b12a890b6a3c8ca56a0 is 50, key is test_row_0/C:col10/1733629275424/Put/seqid=0 2024-12-08T03:41:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742129_1305 (size=13085) 2024-12-08T03:41:16,767 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/9d3fca55b3194b12a890b6a3c8ca56a0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9d3fca55b3194b12a890b6a3c8ca56a0 2024-12-08T03:41:16,772 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/C of 8e27db14233afe9225314bddd051f0ac into 9d3fca55b3194b12a890b6a3c8ca56a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:16,772 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:16,772 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/C, priority=13, startTime=1733629276674; duration=0sec 2024-12-08T03:41:16,772 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:16,772 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:C 2024-12-08T03:41:16,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:41:16,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:16,815 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:41:16,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/086ca3f50cd24ca7af6b8648789b4a2a is 50, key is test_row_0/A:col10/1733629275450/Put/seqid=0 2024-12-08T03:41:16,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742130_1306 (size=12301) 2024-12-08T03:41:16,823 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/086ca3f50cd24ca7af6b8648789b4a2a 2024-12-08T03:41:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/09bdeb95e69744d0bd37f7de9f70b332 is 50, key is test_row_0/B:col10/1733629275450/Put/seqid=0 2024-12-08T03:41:16,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742131_1307 (size=12301) 2024-12-08T03:41:16,838 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/09bdeb95e69744d0bd37f7de9f70b332 2024-12-08T03:41:16,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/e31fcbe828324425881c7734095ee4cf is 50, key is test_row_0/C:col10/1733629275450/Put/seqid=0 2024-12-08T03:41:16,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742132_1308 (size=12301) 2024-12-08T03:41:17,098 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/5c2708380cff41f68cdabc4af789ab4c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/5c2708380cff41f68cdabc4af789ab4c 2024-12-08T03:41:17,101 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e27db14233afe9225314bddd051f0ac/A of 8e27db14233afe9225314bddd051f0ac into 5c2708380cff41f68cdabc4af789ab4c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:17,102 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:17,102 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., storeName=8e27db14233afe9225314bddd051f0ac/A, priority=13, startTime=1733629276673; duration=0sec 2024-12-08T03:41:17,102 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:17,102 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e27db14233afe9225314bddd051f0ac:A 2024-12-08T03:41:17,254 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/e31fcbe828324425881c7734095ee4cf 2024-12-08T03:41:17,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/086ca3f50cd24ca7af6b8648789b4a2a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/086ca3f50cd24ca7af6b8648789b4a2a 2024-12-08T03:41:17,299 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/086ca3f50cd24ca7af6b8648789b4a2a, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T03:41:17,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/09bdeb95e69744d0bd37f7de9f70b332 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/09bdeb95e69744d0bd37f7de9f70b332 2024-12-08T03:41:17,303 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/09bdeb95e69744d0bd37f7de9f70b332, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T03:41:17,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/e31fcbe828324425881c7734095ee4cf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/e31fcbe828324425881c7734095ee4cf 2024-12-08T03:41:17,308 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/e31fcbe828324425881c7734095ee4cf, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T03:41:17,309 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 494ms, sequenceid=351, compaction requested=false 2024-12-08T03:41:17,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 8e27db14233afe9225314bddd051f0ac: 2024-12-08T03:41:17,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:17,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-08T03:41:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-08T03:41:17,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-08T03:41:17,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-12-08T03:41:17,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8730 sec 2024-12-08T03:41:17,463 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-12-08T03:41:17,463 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-12-08T03:41:17,463 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,463 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,464 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,464 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-12-08T03:41:17,464 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-12-08T03:41:17,464 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,464 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,464 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-12-08T03:41:17,464 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,464 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,464 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:41:17,590 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:17,604 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-12-08T03:41:17,604 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-12-08T03:41:17,604 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,604 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,604 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,604 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,605 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,605 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,611 INFO [Thread-1098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1098. 2024-12-08T03:41:17,611 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-12-08T03:41:17,611 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-12-08T03:41:17,611 DEBUG [Thread-1098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,611 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,611 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:17,611 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,611 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,611 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,611 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,611 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,611 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5614 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5566 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5501 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5607 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5561 2024-12-08T03:41:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:41:17,612 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:41:17,612 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:41:17,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:17,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:17,613 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:41:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:41:17,616 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629277616"}]},"ts":"1733629277616"} 2024-12-08T03:41:17,618 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:41:17,618 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:41:17,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:41:17,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, UNASSIGN}] 2024-12-08T03:41:17,621 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, UNASSIGN 2024-12-08T03:41:17,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8e27db14233afe9225314bddd051f0ac, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:17,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, UNASSIGN because future has completed 2024-12-08T03:41:17,624 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:41:17,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:41:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:41:17,776 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:17,776 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:41:17,776 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 8e27db14233afe9225314bddd051f0ac, disabling compactions & flushes 2024-12-08T03:41:17,777 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:17,777 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:17,777 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. after waiting 0 ms 2024-12-08T03:41:17,777 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:17,777 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 8e27db14233afe9225314bddd051f0ac 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-08T03:41:17,780 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/626930527d6a47d0a277e6e0cfcd2aa5 is 50, key is test_row_0/A:col10/1733629277600/Put/seqid=0 2024-12-08T03:41:17,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742133_1309 (size=12301) 2024-12-08T03:41:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:41:18,185 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/626930527d6a47d0a277e6e0cfcd2aa5 2024-12-08T03:41:18,200 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/e0887b646d164038ba035ce9869a3d27 is 50, key is test_row_0/B:col10/1733629277600/Put/seqid=0 2024-12-08T03:41:18,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742134_1310 (size=12301) 2024-12-08T03:41:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:41:18,607 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/e0887b646d164038ba035ce9869a3d27 2024-12-08T03:41:18,619 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/2dc435cad08443b4b633b7425b377323 is 50, key is test_row_0/C:col10/1733629277600/Put/seqid=0 2024-12-08T03:41:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742135_1311 (size=12301) 2024-12-08T03:41:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:41:19,024 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/2dc435cad08443b4b633b7425b377323 2024-12-08T03:41:19,031 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/A/626930527d6a47d0a277e6e0cfcd2aa5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/626930527d6a47d0a277e6e0cfcd2aa5 2024-12-08T03:41:19,038 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/626930527d6a47d0a277e6e0cfcd2aa5, entries=150, sequenceid=360, filesize=12.0 K 2024-12-08T03:41:19,039 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/B/e0887b646d164038ba035ce9869a3d27 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e0887b646d164038ba035ce9869a3d27 2024-12-08T03:41:19,042 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e0887b646d164038ba035ce9869a3d27, entries=150, sequenceid=360, filesize=12.0 K 2024-12-08T03:41:19,043 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/.tmp/C/2dc435cad08443b4b633b7425b377323 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/2dc435cad08443b4b633b7425b377323 2024-12-08T03:41:19,047 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/2dc435cad08443b4b633b7425b377323, entries=150, sequenceid=360, filesize=12.0 K 2024-12-08T03:41:19,048 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 1271ms, sequenceid=360, compaction requested=true 2024-12-08T03:41:19,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f99d5ab0d08d49df9efacfb1fd24a190, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/3c7fda20898c49d5a09d6a700b2a6537, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/4fe653a65d1d4920b55d88e287d0ec3b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/0ff4c80f749d451bbd178c0fb55d27a2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/969dad91d9ce404b8c6158fc445b3d1a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/04f5ca29dc7940e8b86928c9b17a7a61, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/83a51ac2abe048c0a229976861427fcd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/b227fd05be5b42f9b7ba7b60bef445c8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/938a6f1d7b8a49a1bd8f0657b5c968d0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/844d3cfa261d416f90fc67d18329705a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/38d1910347b74344bfad64f5aee3f665, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94b394a3c7f248fcad868720fd8bf794, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/ac00ffb077724378afee25b6628ec5f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94dd451d70504c72822113d47e556d52, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/e0e1f1c0e7754c5aa1d3d3136d560426, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f616b58058e843fa81631b6e30a45bd7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/d5ba78995d91417a83f2cd94af08bd65, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/fc5dea8d29af4f588942d48d540f69b2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/c1ed9bb794a84804b52d90a25fdc605f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/2411ada05c6d49daafd0d951b30d7125, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/521eb065572a4bdc911e3cda5da9591d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9149162ab89a40a4a033a8ef3f147dda, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9a59db0efadf49568b0b1578bf05252f] to archive 2024-12-08T03:41:19,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:41:19,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f99d5ab0d08d49df9efacfb1fd24a190 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f99d5ab0d08d49df9efacfb1fd24a190 2024-12-08T03:41:19,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/3c7fda20898c49d5a09d6a700b2a6537 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/3c7fda20898c49d5a09d6a700b2a6537 2024-12-08T03:41:19,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/4fe653a65d1d4920b55d88e287d0ec3b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/4fe653a65d1d4920b55d88e287d0ec3b 2024-12-08T03:41:19,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/0ff4c80f749d451bbd178c0fb55d27a2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/0ff4c80f749d451bbd178c0fb55d27a2 2024-12-08T03:41:19,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/969dad91d9ce404b8c6158fc445b3d1a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/969dad91d9ce404b8c6158fc445b3d1a 2024-12-08T03:41:19,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/04f5ca29dc7940e8b86928c9b17a7a61 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/04f5ca29dc7940e8b86928c9b17a7a61 2024-12-08T03:41:19,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/83a51ac2abe048c0a229976861427fcd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/83a51ac2abe048c0a229976861427fcd 2024-12-08T03:41:19,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/b227fd05be5b42f9b7ba7b60bef445c8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/b227fd05be5b42f9b7ba7b60bef445c8 2024-12-08T03:41:19,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/938a6f1d7b8a49a1bd8f0657b5c968d0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/938a6f1d7b8a49a1bd8f0657b5c968d0 2024-12-08T03:41:19,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/844d3cfa261d416f90fc67d18329705a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/844d3cfa261d416f90fc67d18329705a 2024-12-08T03:41:19,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/38d1910347b74344bfad64f5aee3f665 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/38d1910347b74344bfad64f5aee3f665 2024-12-08T03:41:19,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94b394a3c7f248fcad868720fd8bf794 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94b394a3c7f248fcad868720fd8bf794 2024-12-08T03:41:19,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/ac00ffb077724378afee25b6628ec5f0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/ac00ffb077724378afee25b6628ec5f0 2024-12-08T03:41:19,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94dd451d70504c72822113d47e556d52 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/94dd451d70504c72822113d47e556d52 2024-12-08T03:41:19,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/e0e1f1c0e7754c5aa1d3d3136d560426 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/e0e1f1c0e7754c5aa1d3d3136d560426 2024-12-08T03:41:19,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f616b58058e843fa81631b6e30a45bd7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/f616b58058e843fa81631b6e30a45bd7 2024-12-08T03:41:19,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/d5ba78995d91417a83f2cd94af08bd65 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/d5ba78995d91417a83f2cd94af08bd65 2024-12-08T03:41:19,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/fc5dea8d29af4f588942d48d540f69b2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/fc5dea8d29af4f588942d48d540f69b2 2024-12-08T03:41:19,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/c1ed9bb794a84804b52d90a25fdc605f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/c1ed9bb794a84804b52d90a25fdc605f 2024-12-08T03:41:19,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/2411ada05c6d49daafd0d951b30d7125 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/2411ada05c6d49daafd0d951b30d7125 2024-12-08T03:41:19,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/521eb065572a4bdc911e3cda5da9591d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/521eb065572a4bdc911e3cda5da9591d 2024-12-08T03:41:19,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9149162ab89a40a4a033a8ef3f147dda to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9149162ab89a40a4a033a8ef3f147dda 2024-12-08T03:41:19,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9a59db0efadf49568b0b1578bf05252f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/9a59db0efadf49568b0b1578bf05252f 2024-12-08T03:41:19,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7f5489505346439b9afa1fec071bdaf8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/4f404881e6f44f2e9d9f3541a24e4c4d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/dcc340c659914041a272bcc3359a94cb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ea511475f46f464888ed7ff42adff894, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/69a1ac8b04d3404b840519d97de0b14b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0dd288dbc36b4d74b36af20e7f382ef7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/964305c622394b7c8fcd569bdcd38f10, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/491b06e1d0854717839578ab6a8af2a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0bac04ecbcb4490d887f61b04735a897, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/db5644b6193249bd865b4152919ef004, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/d76fe9eb8f19434ba0f041cedabb6d03, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c890568024254f19aab0d5a50d576d9b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/1861590dc7d44b9abb8b2d7d58f9b032, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/cfbe6f4ea42d40fa9df9eb2942432e66, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/75fe0fd0925e4615950d1a8ae8f67c16, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/a836244c9605467b8e107c1fb9fe70ff, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/03902c818cc146d2be339b7ec25cc87e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ef0878f67aa9471a9a12954e215648d6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7506f6339c664b6a89d16f559ed91ee4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e7fc7d4422694570b968af5c6f430eaa, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c91c4bdd61fb4c4ebf688074ea5acabe, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/df11c6dbba394046a88a86309a4ab8eb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/f6299543b25a41f3b9ec19cae8df2cb3] to archive 2024-12-08T03:41:19,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:41:19,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7f5489505346439b9afa1fec071bdaf8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7f5489505346439b9afa1fec071bdaf8 2024-12-08T03:41:19,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/4f404881e6f44f2e9d9f3541a24e4c4d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/4f404881e6f44f2e9d9f3541a24e4c4d 2024-12-08T03:41:19,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/dcc340c659914041a272bcc3359a94cb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/dcc340c659914041a272bcc3359a94cb 2024-12-08T03:41:19,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ea511475f46f464888ed7ff42adff894 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ea511475f46f464888ed7ff42adff894 2024-12-08T03:41:19,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/69a1ac8b04d3404b840519d97de0b14b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/69a1ac8b04d3404b840519d97de0b14b 2024-12-08T03:41:19,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0dd288dbc36b4d74b36af20e7f382ef7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0dd288dbc36b4d74b36af20e7f382ef7 2024-12-08T03:41:19,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/964305c622394b7c8fcd569bdcd38f10 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/964305c622394b7c8fcd569bdcd38f10 2024-12-08T03:41:19,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/491b06e1d0854717839578ab6a8af2a7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/491b06e1d0854717839578ab6a8af2a7 2024-12-08T03:41:19,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0bac04ecbcb4490d887f61b04735a897 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0bac04ecbcb4490d887f61b04735a897 2024-12-08T03:41:19,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/db5644b6193249bd865b4152919ef004 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/db5644b6193249bd865b4152919ef004 2024-12-08T03:41:19,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/d76fe9eb8f19434ba0f041cedabb6d03 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/d76fe9eb8f19434ba0f041cedabb6d03 2024-12-08T03:41:19,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c890568024254f19aab0d5a50d576d9b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c890568024254f19aab0d5a50d576d9b 2024-12-08T03:41:19,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/1861590dc7d44b9abb8b2d7d58f9b032 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/1861590dc7d44b9abb8b2d7d58f9b032 2024-12-08T03:41:19,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/cfbe6f4ea42d40fa9df9eb2942432e66 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/cfbe6f4ea42d40fa9df9eb2942432e66 2024-12-08T03:41:19,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/75fe0fd0925e4615950d1a8ae8f67c16 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/75fe0fd0925e4615950d1a8ae8f67c16 2024-12-08T03:41:19,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/a836244c9605467b8e107c1fb9fe70ff to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/a836244c9605467b8e107c1fb9fe70ff 2024-12-08T03:41:19,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/03902c818cc146d2be339b7ec25cc87e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/03902c818cc146d2be339b7ec25cc87e 2024-12-08T03:41:19,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ef0878f67aa9471a9a12954e215648d6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/ef0878f67aa9471a9a12954e215648d6 2024-12-08T03:41:19,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7506f6339c664b6a89d16f559ed91ee4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/7506f6339c664b6a89d16f559ed91ee4 2024-12-08T03:41:19,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e7fc7d4422694570b968af5c6f430eaa to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e7fc7d4422694570b968af5c6f430eaa 2024-12-08T03:41:19,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c91c4bdd61fb4c4ebf688074ea5acabe to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/c91c4bdd61fb4c4ebf688074ea5acabe 2024-12-08T03:41:19,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/df11c6dbba394046a88a86309a4ab8eb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/df11c6dbba394046a88a86309a4ab8eb 2024-12-08T03:41:19,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/f6299543b25a41f3b9ec19cae8df2cb3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/f6299543b25a41f3b9ec19cae8df2cb3 2024-12-08T03:41:19,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/bba714844bab42e89d95154e3ec74c9c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d564a7f7f5e8415db876a4f74405e73a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/dfee553dc67047fda2f78b0ebf6914a1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/f02c707fece74626b220b5ddd8242805, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04f0b6e95af44e68a968faa8bb8eb648, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/c7e19aef83df4bd2bf5adbb04ee13fd9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/271938acab444521b8fb73bef66e93af, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/1688fb0e12d448b6a15101551b309451, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d97ed878fb6a4a10a84930b1a84d95b0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04949aca2af84aec858f142c619fa4bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/80d36d924d6f4fc2b37546e36217d5a5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cef2283bf72446a4affaedfb6286c7cb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/379e1ecbfdf74046874096634963eb3f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ff1584dcb8c840ecb675042ac7683580, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9ac991a5b239437e991b7bc769b9c1c1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ea36d494b05a4759b668d7ccc9a8a087, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/452244b916cf467c9b4dc21f5dfde7da, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8943b3490f4b44e3a6b0137afe8f04bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/74104f1a10264085bbd5d1e01ac34a8d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cfccc0ef28684b48a73ac86db823f74d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8d98a7640b5e4e59beb23ebfcc977b90, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/007f63c1e2cc45e4b04bcf30cd2ddb24, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/13ad1b89e6374e4eb76112495ef34699] to archive 2024-12-08T03:41:19,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:41:19,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/bba714844bab42e89d95154e3ec74c9c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/bba714844bab42e89d95154e3ec74c9c 2024-12-08T03:41:19,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d564a7f7f5e8415db876a4f74405e73a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d564a7f7f5e8415db876a4f74405e73a 2024-12-08T03:41:19,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/dfee553dc67047fda2f78b0ebf6914a1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/dfee553dc67047fda2f78b0ebf6914a1 2024-12-08T03:41:19,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/f02c707fece74626b220b5ddd8242805 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/f02c707fece74626b220b5ddd8242805 2024-12-08T03:41:19,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04f0b6e95af44e68a968faa8bb8eb648 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04f0b6e95af44e68a968faa8bb8eb648 2024-12-08T03:41:19,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/c7e19aef83df4bd2bf5adbb04ee13fd9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/c7e19aef83df4bd2bf5adbb04ee13fd9 2024-12-08T03:41:19,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/271938acab444521b8fb73bef66e93af to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/271938acab444521b8fb73bef66e93af 2024-12-08T03:41:19,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/1688fb0e12d448b6a15101551b309451 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/1688fb0e12d448b6a15101551b309451 2024-12-08T03:41:19,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d97ed878fb6a4a10a84930b1a84d95b0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/d97ed878fb6a4a10a84930b1a84d95b0 2024-12-08T03:41:19,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04949aca2af84aec858f142c619fa4bc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/04949aca2af84aec858f142c619fa4bc 2024-12-08T03:41:19,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/80d36d924d6f4fc2b37546e36217d5a5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/80d36d924d6f4fc2b37546e36217d5a5 2024-12-08T03:41:19,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cef2283bf72446a4affaedfb6286c7cb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cef2283bf72446a4affaedfb6286c7cb 2024-12-08T03:41:19,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/379e1ecbfdf74046874096634963eb3f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/379e1ecbfdf74046874096634963eb3f 2024-12-08T03:41:19,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ff1584dcb8c840ecb675042ac7683580 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ff1584dcb8c840ecb675042ac7683580 2024-12-08T03:41:19,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9ac991a5b239437e991b7bc769b9c1c1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9ac991a5b239437e991b7bc769b9c1c1 2024-12-08T03:41:19,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ea36d494b05a4759b668d7ccc9a8a087 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/ea36d494b05a4759b668d7ccc9a8a087 2024-12-08T03:41:19,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/452244b916cf467c9b4dc21f5dfde7da to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/452244b916cf467c9b4dc21f5dfde7da 2024-12-08T03:41:19,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8943b3490f4b44e3a6b0137afe8f04bc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8943b3490f4b44e3a6b0137afe8f04bc 2024-12-08T03:41:19,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/74104f1a10264085bbd5d1e01ac34a8d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/74104f1a10264085bbd5d1e01ac34a8d 2024-12-08T03:41:19,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cfccc0ef28684b48a73ac86db823f74d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/cfccc0ef28684b48a73ac86db823f74d 2024-12-08T03:41:19,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8d98a7640b5e4e59beb23ebfcc977b90 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/8d98a7640b5e4e59beb23ebfcc977b90 2024-12-08T03:41:19,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/007f63c1e2cc45e4b04bcf30cd2ddb24 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/007f63c1e2cc45e4b04bcf30cd2ddb24 2024-12-08T03:41:19,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/13ad1b89e6374e4eb76112495ef34699 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/13ad1b89e6374e4eb76112495ef34699 2024-12-08T03:41:19,108 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/recovered.edits/363.seqid, newMaxSeqId=363, maxSeqId=1 2024-12-08T03:41:19,108 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. 2024-12-08T03:41:19,108 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 8e27db14233afe9225314bddd051f0ac: Waiting for close lock at 1733629277776Running coprocessor pre-close hooks at 1733629277776Disabling compacts and flushes for region at 1733629277776Disabling writes for close at 1733629277777 (+1 ms)Obtaining lock to block concurrent updates at 1733629277777Preparing flush snapshotting stores in 8e27db14233afe9225314bddd051f0ac at 1733629277777Finished memstore snapshotting TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733629277777Flushing stores of TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac. at 1733629277777Flushing 8e27db14233afe9225314bddd051f0ac/A: creating writer at 1733629277777Flushing 8e27db14233afe9225314bddd051f0ac/A: appending metadata at 1733629277780 (+3 ms)Flushing 8e27db14233afe9225314bddd051f0ac/A: closing flushed file at 1733629277780Flushing 8e27db14233afe9225314bddd051f0ac/B: creating writer at 1733629278194 (+414 ms)Flushing 8e27db14233afe9225314bddd051f0ac/B: appending metadata at 1733629278199 (+5 ms)Flushing 8e27db14233afe9225314bddd051f0ac/B: closing flushed file at 1733629278199Flushing 8e27db14233afe9225314bddd051f0ac/C: creating writer at 1733629278616 (+417 ms)Flushing 8e27db14233afe9225314bddd051f0ac/C: appending metadata at 1733629278618 (+2 ms)Flushing 8e27db14233afe9225314bddd051f0ac/C: closing flushed file at 1733629278618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e539714: reopening flushed file at 1733629279030 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4869c029: reopening flushed file at 1733629279038 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7923e0a5: reopening flushed file at 1733629279043 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8e27db14233afe9225314bddd051f0ac in 1271ms, sequenceid=360, compaction requested=true at 1733629279048 (+5 ms)Writing region close event to WAL at 1733629279106 (+58 ms)Running coprocessor post-close hooks at 1733629279108 (+2 ms)Closed at 1733629279108 2024-12-08T03:41:19,110 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:19,110 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8e27db14233afe9225314bddd051f0ac, regionState=CLOSED 2024-12-08T03:41:19,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:41:19,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-08T03:41:19,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 8e27db14233afe9225314bddd051f0ac, server=32df4b4bb439,42629,1733629194165 in 1.4880 sec 2024-12-08T03:41:19,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-08T03:41:19,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e27db14233afe9225314bddd051f0ac, UNASSIGN in 1.4940 sec 2024-12-08T03:41:19,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-08T03:41:19,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4980 sec 2024-12-08T03:41:19,118 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629279118"}]},"ts":"1733629279118"} 2024-12-08T03:41:19,120 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:41:19,120 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:41:19,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5070 sec 2024-12-08T03:41:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:41:19,760 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:19,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:41:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,766 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T03:41:19,768 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,772 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:19,775 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/recovered.edits] 2024-12-08T03:41:19,778 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/086ca3f50cd24ca7af6b8648789b4a2a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/086ca3f50cd24ca7af6b8648789b4a2a 2024-12-08T03:41:19,780 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/5c2708380cff41f68cdabc4af789ab4c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/5c2708380cff41f68cdabc4af789ab4c 2024-12-08T03:41:19,781 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/626930527d6a47d0a277e6e0cfcd2aa5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/A/626930527d6a47d0a277e6e0cfcd2aa5 2024-12-08T03:41:19,783 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/09bdeb95e69744d0bd37f7de9f70b332 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/09bdeb95e69744d0bd37f7de9f70b332 2024-12-08T03:41:19,785 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0fda8bb1192d41cd82462b80ee845344 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/0fda8bb1192d41cd82462b80ee845344 2024-12-08T03:41:19,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e0887b646d164038ba035ce9869a3d27 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/B/e0887b646d164038ba035ce9869a3d27 2024-12-08T03:41:19,788 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/2dc435cad08443b4b633b7425b377323 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/2dc435cad08443b4b633b7425b377323 2024-12-08T03:41:19,789 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9d3fca55b3194b12a890b6a3c8ca56a0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/9d3fca55b3194b12a890b6a3c8ca56a0 2024-12-08T03:41:19,790 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/e31fcbe828324425881c7734095ee4cf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/C/e31fcbe828324425881c7734095ee4cf 2024-12-08T03:41:19,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/recovered.edits/363.seqid to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac/recovered.edits/363.seqid 2024-12-08T03:41:19,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/8e27db14233afe9225314bddd051f0ac 2024-12-08T03:41:19,793 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:41:19,796 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,799 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:41:19,802 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:41:19,803 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,803 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:41:19,803 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733629279803"}]},"ts":"9223372036854775807"} 2024-12-08T03:41:19,805 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:41:19,805 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8e27db14233afe9225314bddd051f0ac, NAME => 'TestAcidGuarantees,,1733629255246.8e27db14233afe9225314bddd051f0ac.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:41:19,805 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:41:19,805 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733629279805"}]},"ts":"9223372036854775807"} 2024-12-08T03:41:19,807 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:41:19,808 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-12-08T03:41:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T03:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:41:19,879 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:19,891 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241 (was 242), OpenFileDescriptor=457 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 465), ProcessCount=11 (was 11), AvailableMemoryMB=17180 (was 17252) 2024-12-08T03:41:19,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=17179 2024-12-08T03:41:19,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:41:19,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:41:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:19,905 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:41:19,905 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:19,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-12-08T03:41:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T03:41:19,906 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:41:19,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742136_1312 (size=959) 2024-12-08T03:41:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T03:41:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T03:41:20,317 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:41:20,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742137_1313 (size=53) 2024-12-08T03:41:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T03:41:20,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:41:20,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 440aad18abead1c4be188777b931aa43, disabling compactions & flushes 2024-12-08T03:41:20,732 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:20,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:20,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. after waiting 0 ms 2024-12-08T03:41:20,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:20,732 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:20,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 440aad18abead1c4be188777b931aa43: Waiting for close lock at 1733629280732Disabling compacts and flushes for region at 1733629280732Disabling writes for close at 1733629280732Writing region close event to WAL at 1733629280732Closed at 1733629280732 2024-12-08T03:41:20,734 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:41:20,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733629280734"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733629280734"}]},"ts":"1733629280734"} 2024-12-08T03:41:20,737 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:41:20,738 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:41:20,738 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629280738"}]},"ts":"1733629280738"} 2024-12-08T03:41:20,740 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:41:20,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, ASSIGN}] 2024-12-08T03:41:20,742 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, ASSIGN 2024-12-08T03:41:20,744 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, ASSIGN; state=OFFLINE, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=false 2024-12-08T03:41:20,895 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:20,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, ASSIGN because future has completed 2024-12-08T03:41:20,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:41:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T03:41:21,055 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:21,055 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:41:21,055 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,055 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:41:21,055 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,055 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,056 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,057 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 440aad18abead1c4be188777b931aa43 columnFamilyName A 2024-12-08T03:41:21,057 DEBUG [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:21,058 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(327): Store=440aad18abead1c4be188777b931aa43/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:21,058 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,059 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 440aad18abead1c4be188777b931aa43 columnFamilyName B 2024-12-08T03:41:21,059 DEBUG [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:21,059 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(327): Store=440aad18abead1c4be188777b931aa43/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:21,059 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,060 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 440aad18abead1c4be188777b931aa43 columnFamilyName C 2024-12-08T03:41:21,060 DEBUG [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:21,061 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(327): Store=440aad18abead1c4be188777b931aa43/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:21,061 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,061 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,062 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,062 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,062 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,063 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:41:21,064 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,065 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:41:21,066 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 440aad18abead1c4be188777b931aa43; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65319755, jitterRate=-0.026659801602363586}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:41:21,066 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:21,066 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 440aad18abead1c4be188777b931aa43: Running coprocessor pre-open hook at 1733629281055Writing region info on filesystem at 1733629281055Initializing all the Stores at 1733629281056 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629281056Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629281056Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629281056Cleaning up temporary data from old regions at 1733629281062 (+6 ms)Running coprocessor post-open hooks at 1733629281066 (+4 ms)Region opened successfully at 1733629281066 2024-12-08T03:41:21,067 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., pid=89, masterSystemTime=1733629281052 2024-12-08T03:41:21,068 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:21,068 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:21,069 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:21,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:41:21,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-08T03:41:21,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 in 172 msec 2024-12-08T03:41:21,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-08T03:41:21,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, ASSIGN in 331 msec 2024-12-08T03:41:21,074 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:41:21,074 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629281074"}]},"ts":"1733629281074"} 2024-12-08T03:41:21,075 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:41:21,075 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:41:21,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-08T03:41:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T03:41:22,050 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:22,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:22,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:22,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:22,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f465764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:22,060 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99a5865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:22,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:22,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:22,063 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:41:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:41:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f9d5a7d 2024-12-08T03:41:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:41:22,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:41:22,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:41:22,067 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:41:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:22,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742138_1314 (size=995) 2024-12-08T03:41:22,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:41:22,481 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-08T03:41:22,481 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-08T03:41:22,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:41:22,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, REOPEN/MOVE}] 2024-12-08T03:41:22,488 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, REOPEN/MOVE 2024-12-08T03:41:22,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:22,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, REOPEN/MOVE because future has completed 2024-12-08T03:41:22,490 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:41:22,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:41:22,643 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:41:22,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 440aad18abead1c4be188777b931aa43, disabling compactions & flushes 2024-12-08T03:41:22,643 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. after waiting 0 ms 2024-12-08T03:41:22,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,652 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T03:41:22,653 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,653 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 440aad18abead1c4be188777b931aa43: Waiting for close lock at 1733629282643Running coprocessor pre-close hooks at 1733629282643Disabling compacts and flushes for region at 1733629282643Disabling writes for close at 1733629282643Writing region close event to WAL at 1733629282644 (+1 ms)Running coprocessor post-close hooks at 1733629282653 (+9 ms)Closed at 1733629282653 2024-12-08T03:41:22,653 WARN [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 440aad18abead1c4be188777b931aa43 to self. 2024-12-08T03:41:22,656 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,657 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=CLOSED 2024-12-08T03:41:22,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:41:22,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-08T03:41:22,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 in 170 msec 2024-12-08T03:41:22,664 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=true 2024-12-08T03:41:22,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:22,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, REOPEN/MOVE because future has completed 2024-12-08T03:41:22,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:41:22,977 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,977 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:41:22,977 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,977 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:41:22,978 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,978 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,979 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,980 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 440aad18abead1c4be188777b931aa43 columnFamilyName A 2024-12-08T03:41:22,982 DEBUG [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:22,982 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(327): Store=440aad18abead1c4be188777b931aa43/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:22,983 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,984 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 440aad18abead1c4be188777b931aa43 columnFamilyName B 2024-12-08T03:41:22,984 DEBUG [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:22,984 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(327): Store=440aad18abead1c4be188777b931aa43/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:22,984 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,985 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 440aad18abead1c4be188777b931aa43 columnFamilyName C 2024-12-08T03:41:22,985 DEBUG [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:22,986 INFO [StoreOpener-440aad18abead1c4be188777b931aa43-1 {}] regionserver.HStore(327): Store=440aad18abead1c4be188777b931aa43/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:22,986 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,987 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,988 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,989 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,989 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,990 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:41:22,991 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,992 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 440aad18abead1c4be188777b931aa43; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75424244, jitterRate=0.12390881776809692}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:41:22,992 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:22,992 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 440aad18abead1c4be188777b931aa43: Running coprocessor pre-open hook at 1733629282978Writing region info on filesystem at 1733629282978Initializing all the Stores at 1733629282979 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629282979Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629282979Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629282979Cleaning up temporary data from old regions at 1733629282989 (+10 ms)Running coprocessor post-open hooks at 1733629282992 (+3 ms)Region opened successfully at 1733629282992 2024-12-08T03:41:22,993 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., pid=94, masterSystemTime=1733629282970 2024-12-08T03:41:22,994 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,994 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:22,995 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=OPEN, openSeqNum=5, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:22,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:41:22,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-12-08T03:41:22,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 in 180 msec 2024-12-08T03:41:23,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-08T03:41:23,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, REOPEN/MOVE in 512 msec 2024-12-08T03:41:23,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-08T03:41:23,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-12-08T03:41:23,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-12-08T03:41:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-08T03:41:23,005 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:23,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,006 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1828a07d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583be2f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,010 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b1f0cc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,011 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16cda4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ada1567, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6784fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@805557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a07f565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,019 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d694d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df8c30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,024 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758c2766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f6545e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,027 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337d2835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,028 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d75310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce9c458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a54c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1376c43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,035 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cd9afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:23,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:23,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de19b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:23,039 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275aa2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:23,041 DEBUG [Thread-1456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,042 DEBUG [Thread-1458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,042 DEBUG [Thread-1445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,043 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,044 DEBUG [Thread-1454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,048 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,057 DEBUG [Thread-1449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:23,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-08T03:41:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:23,067 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:23,067 DEBUG [Thread-1443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,067 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:23,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,071 DEBUG [Thread-1447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,072 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,073 DEBUG [Thread-1460 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:23,073 DEBUG [Thread-1460 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:23,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:41:23,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208115a4d011f514cfaa401ea91100f8d42_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629283054/Put/seqid=0 2024-12-08T03:41:23,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742139_1315 (size=14594) 2024-12-08T03:41:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629343119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629343119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629343119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629343120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629343126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:23,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629343238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629343238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629343239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629343239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629343239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:23,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629343448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629343449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629343449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629343449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629343449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:23,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208115a4d011f514cfaa401ea91100f8d42_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208115a4d011f514cfaa401ea91100f8d42_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:23,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/e1ebb83081d64ca694126211e5aab1c6, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:23,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/e1ebb83081d64ca694126211e5aab1c6 is 175, key is test_row_0/A:col10/1733629283054/Put/seqid=0 2024-12-08T03:41:23,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742140_1316 (size=39549) 2024-12-08T03:41:23,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:23,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629343758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629343758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629343758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629343758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629343759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:23,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/e1ebb83081d64ca694126211e5aab1c6 2024-12-08T03:41:23,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/fec7a6862c0847d59f24a96e67de3e7a is 50, key is test_row_0/B:col10/1733629283054/Put/seqid=0 2024-12-08T03:41:23,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742141_1317 (size=12001) 2024-12-08T03:41:23,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:23,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:24,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629344268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629344268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629344268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629344268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629344278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:24,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:24,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:24,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/fec7a6862c0847d59f24a96e67de3e7a 2024-12-08T03:41:24,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/d559749b3ea842beb23d9c59b220c5a7 is 50, key is test_row_0/C:col10/1733629283054/Put/seqid=0 2024-12-08T03:41:24,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742142_1318 (size=12001) 2024-12-08T03:41:24,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/d559749b3ea842beb23d9c59b220c5a7 2024-12-08T03:41:24,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/e1ebb83081d64ca694126211e5aab1c6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6 2024-12-08T03:41:24,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6, entries=200, sequenceid=15, filesize=38.6 K 2024-12-08T03:41:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/fec7a6862c0847d59f24a96e67de3e7a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/fec7a6862c0847d59f24a96e67de3e7a 2024-12-08T03:41:24,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/fec7a6862c0847d59f24a96e67de3e7a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:41:24,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/d559749b3ea842beb23d9c59b220c5a7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/d559749b3ea842beb23d9c59b220c5a7 2024-12-08T03:41:24,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/d559749b3ea842beb23d9c59b220c5a7, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:41:24,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 440aad18abead1c4be188777b931aa43 in 1319ms, sequenceid=15, compaction requested=false 2024-12-08T03:41:24,380 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:41:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T03:41:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:24,443 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:41:24,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b9ae548debc04374ab9afae2f00b31e2_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629283094/Put/seqid=0 2024-12-08T03:41:24,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742143_1319 (size=12154) 2024-12-08T03:41:24,629 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:41:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:24,857 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b9ae548debc04374ab9afae2f00b31e2_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b9ae548debc04374ab9afae2f00b31e2_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d178ad80a7de49c59d69196a95f482ed, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d178ad80a7de49c59d69196a95f482ed is 175, key is test_row_0/A:col10/1733629283094/Put/seqid=0 2024-12-08T03:41:24,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742144_1320 (size=30955) 2024-12-08T03:41:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:25,263 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d178ad80a7de49c59d69196a95f482ed 2024-12-08T03:41:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/d93ebd93682e4d2ebad20006d591454b is 50, key is test_row_0/B:col10/1733629283094/Put/seqid=0 2024-12-08T03:41:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742145_1321 (size=12001) 2024-12-08T03:41:25,275 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/d93ebd93682e4d2ebad20006d591454b 2024-12-08T03:41:25,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0bd945ccec294b26abce4ddc8d49e2e8 is 50, key is test_row_0/C:col10/1733629283094/Put/seqid=0 2024-12-08T03:41:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:25,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742146_1322 (size=12001) 2024-12-08T03:41:25,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629345292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629345293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629345295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629345296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629345298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,305 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:25,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629345409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629345409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629345409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629345409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629345618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629345619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629345619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629345619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,689 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0bd945ccec294b26abce4ddc8d49e2e8 2024-12-08T03:41:25,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d178ad80a7de49c59d69196a95f482ed as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed 2024-12-08T03:41:25,700 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed, entries=150, sequenceid=40, filesize=30.2 K 2024-12-08T03:41:25,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/d93ebd93682e4d2ebad20006d591454b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d93ebd93682e4d2ebad20006d591454b 2024-12-08T03:41:25,703 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d93ebd93682e4d2ebad20006d591454b, entries=150, sequenceid=40, filesize=11.7 K 2024-12-08T03:41:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0bd945ccec294b26abce4ddc8d49e2e8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0bd945ccec294b26abce4ddc8d49e2e8 2024-12-08T03:41:25,707 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0bd945ccec294b26abce4ddc8d49e2e8, entries=150, sequenceid=40, filesize=11.7 K 2024-12-08T03:41:25,708 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 440aad18abead1c4be188777b931aa43 in 1265ms, sequenceid=40, compaction requested=false 2024-12-08T03:41:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-08T03:41:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-08T03:41:25,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-08T03:41:25,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-12-08T03:41:25,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.6460 sec 2024-12-08T03:41:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:25,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:25,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:41:25,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e3db6f562ef14260b858cc9c61d8aec9_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:25,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742147_1323 (size=12154) 2024-12-08T03:41:25,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:25,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e3db6f562ef14260b858cc9c61d8aec9_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3db6f562ef14260b858cc9c61d8aec9_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:25,949 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/c02e650619b34b509600f861a56a3bc4, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:25,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/c02e650619b34b509600f861a56a3bc4 is 175, key is test_row_0/A:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:25,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742148_1324 (size=30955) 2024-12-08T03:41:25,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/c02e650619b34b509600f861a56a3bc4 2024-12-08T03:41:25,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/9003850dae5249a0a4581f615cf29241 is 50, key is test_row_0/B:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742149_1325 (size=12001) 2024-12-08T03:41:25,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629345968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629345969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629345971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629345972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629346078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629346078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629346088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629346088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629346288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629346288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629346298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629346298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/9003850dae5249a0a4581f615cf29241 2024-12-08T03:41:26,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/52848e5b4afa4a79ad9826054d4ff772 is 50, key is test_row_0/C:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:26,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742150_1326 (size=12001) 2024-12-08T03:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629346598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629346598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629346609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629346609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:26,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/52848e5b4afa4a79ad9826054d4ff772 2024-12-08T03:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/c02e650619b34b509600f861a56a3bc4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4 2024-12-08T03:41:26,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4, entries=150, sequenceid=53, filesize=30.2 K 2024-12-08T03:41:26,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/9003850dae5249a0a4581f615cf29241 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/9003850dae5249a0a4581f615cf29241 2024-12-08T03:41:26,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/9003850dae5249a0a4581f615cf29241, entries=150, sequenceid=53, filesize=11.7 K 2024-12-08T03:41:26,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/52848e5b4afa4a79ad9826054d4ff772 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/52848e5b4afa4a79ad9826054d4ff772 2024-12-08T03:41:26,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/52848e5b4afa4a79ad9826054d4ff772, entries=150, sequenceid=53, filesize=11.7 K 2024-12-08T03:41:26,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 440aad18abead1c4be188777b931aa43 in 857ms, sequenceid=53, compaction requested=true 2024-12-08T03:41:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:26,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:26,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:26,790 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:26,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:26,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:26,790 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:26,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:26,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:26,790 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:26,790 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:26,790 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/B is initiating minor compaction (all files) 2024-12-08T03:41:26,790 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/A is initiating minor compaction (all files) 2024-12-08T03:41:26,791 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/A in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:26,791 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/B in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:26,791 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=99.1 K 2024-12-08T03:41:26,791 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/fec7a6862c0847d59f24a96e67de3e7a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d93ebd93682e4d2ebad20006d591454b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/9003850dae5249a0a4581f615cf29241] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=35.2 K 2024-12-08T03:41:26,791 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:26,791 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4] 2024-12-08T03:41:26,791 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ebb83081d64ca694126211e5aab1c6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629283054 2024-12-08T03:41:26,791 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting fec7a6862c0847d59f24a96e67de3e7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629283054 2024-12-08T03:41:26,791 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d93ebd93682e4d2ebad20006d591454b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629283084 2024-12-08T03:41:26,791 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d178ad80a7de49c59d69196a95f482ed, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629283084 2024-12-08T03:41:26,792 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 9003850dae5249a0a4581f615cf29241, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733629285292 2024-12-08T03:41:26,792 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting c02e650619b34b509600f861a56a3bc4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733629285292 2024-12-08T03:41:26,796 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:26,797 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:26,797 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/482362db451b4991b82fe1558a09c064 is 50, key is test_row_0/B:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:26,803 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208e2964af3f4cb4d0dac1348822c18b81b_440aad18abead1c4be188777b931aa43 store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:26,805 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208e2964af3f4cb4d0dac1348822c18b81b_440aad18abead1c4be188777b931aa43, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:26,805 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e2964af3f4cb4d0dac1348822c18b81b_440aad18abead1c4be188777b931aa43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742151_1327 (size=12104) 2024-12-08T03:41:26,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742152_1328 (size=4469) 2024-12-08T03:41:26,817 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#A#compaction#280 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:26,817 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/e3825f8769324ffcb51d072f0e778dc5 is 175, key is test_row_0/A:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:26,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742153_1329 (size=31058) 2024-12-08T03:41:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:27,112 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:27,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:41:27,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d6d381e90a964e1b87e26f52f2d4476a_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629285955/Put/seqid=0 2024-12-08T03:41:27,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742154_1330 (size=14594) 2024-12-08T03:41:27,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629347124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629347128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629347129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629347129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,216 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/482362db451b4991b82fe1558a09c064 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/482362db451b4991b82fe1558a09c064 2024-12-08T03:41:27,220 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/B of 440aad18abead1c4be188777b931aa43 into 482362db451b4991b82fe1558a09c064(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:27,220 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:27,220 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/B, priority=13, startTime=1733629286790; duration=0sec 2024-12-08T03:41:27,220 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:27,220 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:B 2024-12-08T03:41:27,221 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:27,221 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:27,221 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/C is initiating minor compaction (all files) 2024-12-08T03:41:27,221 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/C in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,221 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/d559749b3ea842beb23d9c59b220c5a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0bd945ccec294b26abce4ddc8d49e2e8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/52848e5b4afa4a79ad9826054d4ff772] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=35.2 K 2024-12-08T03:41:27,222 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d559749b3ea842beb23d9c59b220c5a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629283054 2024-12-08T03:41:27,222 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd945ccec294b26abce4ddc8d49e2e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733629283084 2024-12-08T03:41:27,222 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 52848e5b4afa4a79ad9826054d4ff772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733629285292 2024-12-08T03:41:27,225 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/e3825f8769324ffcb51d072f0e778dc5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e3825f8769324ffcb51d072f0e778dc5 2024-12-08T03:41:27,229 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:27,229 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/A of 440aad18abead1c4be188777b931aa43 into e3825f8769324ffcb51d072f0e778dc5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:27,229 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:27,229 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/A, priority=13, startTime=1733629286790; duration=0sec 2024-12-08T03:41:27,229 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:27,229 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:A 2024-12-08T03:41:27,229 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/91c48cbe563d43c5b3449b8bfb62adc5 is 50, key is test_row_0/C:col10/1733629285932/Put/seqid=0 2024-12-08T03:41:27,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742155_1331 (size=12104) 2024-12-08T03:41:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T03:41:27,239 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:27,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-08T03:41:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:27,241 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:27,241 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:27,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:27,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629347238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629347239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629347239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629347248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629347328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,330 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:27,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629347448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629347458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629347458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629347458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:27,527 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d6d381e90a964e1b87e26f52f2d4476a_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d6d381e90a964e1b87e26f52f2d4476a_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:27,527 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/ccd07b9d906a425e8f536841c68a89d7, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:27,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/ccd07b9d906a425e8f536841c68a89d7 is 175, key is test_row_0/A:col10/1733629285955/Put/seqid=0 2024-12-08T03:41:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742156_1332 (size=39549) 2024-12-08T03:41:27,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:27,636 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/91c48cbe563d43c5b3449b8bfb62adc5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/91c48cbe563d43c5b3449b8bfb62adc5 2024-12-08T03:41:27,641 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/C of 440aad18abead1c4be188777b931aa43 into 91c48cbe563d43c5b3449b8bfb62adc5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:27,641 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:27,641 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/C, priority=13, startTime=1733629286790; duration=0sec 2024-12-08T03:41:27,641 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:27,641 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:C 2024-12-08T03:41:27,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:27,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629347758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629347768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629347768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629347768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:27,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:27,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:27,932 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/ccd07b9d906a425e8f536841c68a89d7 2024-12-08T03:41:27,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/eab88cb88c4a41ed88de5df67934a344 is 50, key is test_row_0/B:col10/1733629285955/Put/seqid=0 2024-12-08T03:41:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742157_1333 (size=12001) 2024-12-08T03:41:28,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629348268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629348278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:28,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629348288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629348288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:28,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:28,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:28,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/eab88cb88c4a41ed88de5df67934a344 2024-12-08T03:41:28,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/27f88c75d0364bce9882c212c082b602 is 50, key is test_row_0/C:col10/1733629285955/Put/seqid=0 2024-12-08T03:41:28,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742158_1334 (size=12001) 2024-12-08T03:41:28,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/27f88c75d0364bce9882c212c082b602 2024-12-08T03:41:28,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/ccd07b9d906a425e8f536841c68a89d7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7 2024-12-08T03:41:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7, entries=200, sequenceid=77, filesize=38.6 K 2024-12-08T03:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/eab88cb88c4a41ed88de5df67934a344 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/eab88cb88c4a41ed88de5df67934a344 2024-12-08T03:41:28,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/eab88cb88c4a41ed88de5df67934a344, entries=150, sequenceid=77, filesize=11.7 K 2024-12-08T03:41:28,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/27f88c75d0364bce9882c212c082b602 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/27f88c75d0364bce9882c212c082b602 2024-12-08T03:41:28,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/27f88c75d0364bce9882c212c082b602, entries=150, sequenceid=77, filesize=11.7 K 2024-12-08T03:41:28,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 440aad18abead1c4be188777b931aa43 in 1256ms, sequenceid=77, compaction requested=false 2024-12-08T03:41:28,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:28,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T03:41:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:28,461 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:41:28,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080be25b456d98448997de03373f8d9950_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629287128/Put/seqid=0 2024-12-08T03:41:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742159_1335 (size=12154) 2024-12-08T03:41:28,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:28,875 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080be25b456d98448997de03373f8d9950_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080be25b456d98448997de03373f8d9950_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:28,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/67f99be048be44e8b17835779aa46bbb, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:28,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/67f99be048be44e8b17835779aa46bbb is 175, key is test_row_0/A:col10/1733629287128/Put/seqid=0 2024-12-08T03:41:28,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742160_1336 (size=30955) 2024-12-08T03:41:29,280 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/67f99be048be44e8b17835779aa46bbb 2024-12-08T03:41:29,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/e506fb4d64694fc79cc6f61fbacf4cf2 is 50, key is test_row_0/B:col10/1733629287128/Put/seqid=0 2024-12-08T03:41:29,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742161_1337 (size=12001) 2024-12-08T03:41:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629349339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629349340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629349346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629349348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629349458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629349458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629349458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629349468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629349668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629349668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629349668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629349678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,692 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/e506fb4d64694fc79cc6f61fbacf4cf2 2024-12-08T03:41:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/be50aa76951a45f8b9564372f5c58b49 is 50, key is test_row_0/C:col10/1733629287128/Put/seqid=0 2024-12-08T03:41:29,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742162_1338 (size=12001) 2024-12-08T03:41:29,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629349979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629349979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629349988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629349988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,102 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/be50aa76951a45f8b9564372f5c58b49 2024-12-08T03:41:30,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/67f99be048be44e8b17835779aa46bbb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb 2024-12-08T03:41:30,108 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb, entries=150, sequenceid=92, filesize=30.2 K 2024-12-08T03:41:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/e506fb4d64694fc79cc6f61fbacf4cf2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/e506fb4d64694fc79cc6f61fbacf4cf2 2024-12-08T03:41:30,112 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/e506fb4d64694fc79cc6f61fbacf4cf2, entries=150, sequenceid=92, filesize=11.7 K 2024-12-08T03:41:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/be50aa76951a45f8b9564372f5c58b49 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/be50aa76951a45f8b9564372f5c58b49 2024-12-08T03:41:30,116 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/be50aa76951a45f8b9564372f5c58b49, entries=150, sequenceid=92, filesize=11.7 K 2024-12-08T03:41:30,117 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 440aad18abead1c4be188777b931aa43 in 1655ms, sequenceid=92, compaction requested=true 2024-12-08T03:41:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-08T03:41:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-08T03:41:30,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-08T03:41:30,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8770 sec 2024-12-08T03:41:30,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.8810 sec 2024-12-08T03:41:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:30,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:30,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:41:30,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208baa85e1c565c45779d9af29ddb3135fe_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:30,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742163_1339 (size=17034) 2024-12-08T03:41:30,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629350504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629350505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629350508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629350509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629350618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629350618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629350628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629350628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629350828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629350828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629350838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629350838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:30,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:30,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208baa85e1c565c45779d9af29ddb3135fe_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208baa85e1c565c45779d9af29ddb3135fe_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:30,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/94485b4eec9249c39b0dfa4283dd5100, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:30,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/94485b4eec9249c39b0dfa4283dd5100 is 175, key is test_row_0/A:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:30,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742164_1340 (size=48139) 2024-12-08T03:41:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629351138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629351138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629351148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629351148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,317 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/94485b4eec9249c39b0dfa4283dd5100 2024-12-08T03:41:31,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/f0917e2c97594cc9a088bb9526582d5b is 50, key is test_row_0/B:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:31,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742165_1341 (size=12001) 2024-12-08T03:41:31,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629351348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,352 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T03:41:31,408 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:31,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-08T03:41:31,410 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T03:41:31,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:31,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T03:41:31,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T03:41:31,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:31,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:31,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629351648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629351648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629351658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629351658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:31,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T03:41:31,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:31,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:31,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:31,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/f0917e2c97594cc9a088bb9526582d5b 2024-12-08T03:41:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T03:41:31,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/2896425d25a545ec9018a89495951b07 is 50, key is test_row_0/C:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:31,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742166_1342 (size=12001) 2024-12-08T03:41:31,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/2896425d25a545ec9018a89495951b07 2024-12-08T03:41:31,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/94485b4eec9249c39b0dfa4283dd5100 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100 2024-12-08T03:41:31,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100, entries=250, sequenceid=118, filesize=47.0 K 2024-12-08T03:41:31,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/f0917e2c97594cc9a088bb9526582d5b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/f0917e2c97594cc9a088bb9526582d5b 2024-12-08T03:41:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/f0917e2c97594cc9a088bb9526582d5b, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T03:41:31,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/2896425d25a545ec9018a89495951b07 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/2896425d25a545ec9018a89495951b07 2024-12-08T03:41:31,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/2896425d25a545ec9018a89495951b07, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T03:41:31,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 440aad18abead1c4be188777b931aa43 in 1269ms, sequenceid=118, compaction requested=true 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:31,761 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:31,761 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:31,763 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:41:31,763 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:41:31,763 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/A is initiating minor compaction (all files) 2024-12-08T03:41:31,763 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/B is initiating minor compaction (all files) 2024-12-08T03:41:31,763 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/B in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,763 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/A in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,763 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e3825f8769324ffcb51d072f0e778dc5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=146.2 K 2024-12-08T03:41:31,763 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/482362db451b4991b82fe1558a09c064, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/eab88cb88c4a41ed88de5df67934a344, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/e506fb4d64694fc79cc6f61fbacf4cf2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/f0917e2c97594cc9a088bb9526582d5b] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=47.0 K 2024-12-08T03:41:31,763 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,764 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e3825f8769324ffcb51d072f0e778dc5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100] 2024-12-08T03:41:31,764 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 482362db451b4991b82fe1558a09c064, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733629285292 2024-12-08T03:41:31,764 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3825f8769324ffcb51d072f0e778dc5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733629285292 2024-12-08T03:41:31,764 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd07b9d906a425e8f536841c68a89d7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629285955 2024-12-08T03:41:31,764 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting eab88cb88c4a41ed88de5df67934a344, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629285955 2024-12-08T03:41:31,765 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f99be048be44e8b17835779aa46bbb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733629287123 2024-12-08T03:41:31,765 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting e506fb4d64694fc79cc6f61fbacf4cf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733629287123 2024-12-08T03:41:31,765 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94485b4eec9249c39b0dfa4283dd5100, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629289330 2024-12-08T03:41:31,765 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f0917e2c97594cc9a088bb9526582d5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629289330 2024-12-08T03:41:31,773 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:31,775 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#B#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:31,775 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412083603dc5527134c4c92e7e32cff3e91d5_440aad18abead1c4be188777b931aa43 store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:31,776 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/c3e0914d76ac46ad9d12ab388285bd59 is 50, key is test_row_0/B:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:31,777 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412083603dc5527134c4c92e7e32cff3e91d5_440aad18abead1c4be188777b931aa43, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:31,777 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083603dc5527134c4c92e7e32cff3e91d5_440aad18abead1c4be188777b931aa43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:31,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742167_1343 (size=12241) 2024-12-08T03:41:31,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742168_1344 (size=4469) 2024-12-08T03:41:31,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T03:41:31,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:31,868 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-08T03:41:31,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089a2450c5a0ba4f798e2aae63c77ace16_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629290508/Put/seqid=0 2024-12-08T03:41:31,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742169_1345 (size=12154) 2024-12-08T03:41:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T03:41:32,181 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#A#compaction#291 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:32,181 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/b87ea3605b204814a254a81d31c21d32 is 175, key is test_row_0/A:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:32,184 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/c3e0914d76ac46ad9d12ab388285bd59 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/c3e0914d76ac46ad9d12ab388285bd59 2024-12-08T03:41:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742170_1346 (size=31195) 2024-12-08T03:41:32,190 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/b87ea3605b204814a254a81d31c21d32 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/b87ea3605b204814a254a81d31c21d32 2024-12-08T03:41:32,190 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 440aad18abead1c4be188777b931aa43/B of 440aad18abead1c4be188777b931aa43 into c3e0914d76ac46ad9d12ab388285bd59(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:32,190 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:32,190 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/B, priority=12, startTime=1733629291761; duration=0sec 2024-12-08T03:41:32,190 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:32,190 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:B 2024-12-08T03:41:32,190 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:41:32,192 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:41:32,192 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/C is initiating minor compaction (all files) 2024-12-08T03:41:32,192 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/C in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:32,192 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/91c48cbe563d43c5b3449b8bfb62adc5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/27f88c75d0364bce9882c212c082b602, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/be50aa76951a45f8b9564372f5c58b49, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/2896425d25a545ec9018a89495951b07] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=47.0 K 2024-12-08T03:41:32,192 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c48cbe563d43c5b3449b8bfb62adc5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733629285292 2024-12-08T03:41:32,192 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 27f88c75d0364bce9882c212c082b602, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733629285955 2024-12-08T03:41:32,193 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting be50aa76951a45f8b9564372f5c58b49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733629287123 2024-12-08T03:41:32,193 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 2896425d25a545ec9018a89495951b07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629289330 2024-12-08T03:41:32,195 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 440aad18abead1c4be188777b931aa43/A of 440aad18abead1c4be188777b931aa43 into b87ea3605b204814a254a81d31c21d32(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:32,195 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:32,195 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/A, priority=12, startTime=1733629291761; duration=0sec 2024-12-08T03:41:32,195 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:32,195 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:A 2024-12-08T03:41:32,201 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:32,202 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/76590f72bf924813929f0fcdeff56181 is 50, key is test_row_0/C:col10/1733629289330/Put/seqid=0 2024-12-08T03:41:32,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742171_1347 (size=12241) 2024-12-08T03:41:32,225 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/76590f72bf924813929f0fcdeff56181 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/76590f72bf924813929f0fcdeff56181 2024-12-08T03:41:32,231 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 440aad18abead1c4be188777b931aa43/C of 440aad18abead1c4be188777b931aa43 into 76590f72bf924813929f0fcdeff56181(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:32,231 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:32,231 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/C, priority=12, startTime=1733629291761; duration=0sec 2024-12-08T03:41:32,231 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:32,231 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:C 2024-12-08T03:41:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:32,282 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089a2450c5a0ba4f798e2aae63c77ace16_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089a2450c5a0ba4f798e2aae63c77ace16_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:32,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/5f749ade44054e3a841ba01b359732a4, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:32,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/5f749ade44054e3a841ba01b359732a4 is 175, key is test_row_0/A:col10/1733629290508/Put/seqid=0 2024-12-08T03:41:32,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742172_1348 (size=30955) 2024-12-08T03:41:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T03:41:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:32,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:32,692 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/5f749ade44054e3a841ba01b359732a4 2024-12-08T03:41:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/2617a73914c949d2aea1107c0b939853 is 50, key is test_row_0/B:col10/1733629290508/Put/seqid=0 2024-12-08T03:41:32,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742173_1349 (size=12001) 2024-12-08T03:41:32,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629352701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629352705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629352708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629352709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629352818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629352818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629352828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629352828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629353028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629353028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629353038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629353038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,104 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/2617a73914c949d2aea1107c0b939853 2024-12-08T03:41:33,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/a2257df6432c49808182515a2e748f47 is 50, key is test_row_0/C:col10/1733629290508/Put/seqid=0 2024-12-08T03:41:33,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742174_1350 (size=12001) 2024-12-08T03:41:33,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629353338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629353338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629353348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629353348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,518 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/a2257df6432c49808182515a2e748f47 2024-12-08T03:41:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/5f749ade44054e3a841ba01b359732a4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4 2024-12-08T03:41:33,524 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4, entries=150, sequenceid=128, filesize=30.2 K 2024-12-08T03:41:33,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/2617a73914c949d2aea1107c0b939853 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/2617a73914c949d2aea1107c0b939853 2024-12-08T03:41:33,527 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/2617a73914c949d2aea1107c0b939853, entries=150, sequenceid=128, filesize=11.7 K 2024-12-08T03:41:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/a2257df6432c49808182515a2e748f47 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a2257df6432c49808182515a2e748f47 2024-12-08T03:41:33,531 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a2257df6432c49808182515a2e748f47, entries=150, sequenceid=128, filesize=11.7 K 2024-12-08T03:41:33,532 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 440aad18abead1c4be188777b931aa43 in 1663ms, sequenceid=128, compaction requested=false 2024-12-08T03:41:33,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:33,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:33,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-08T03:41:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-08T03:41:33,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-08T03:41:33,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1220 sec 2024-12-08T03:41:33,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1250 sec 2024-12-08T03:41:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T03:41:33,558 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:33,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-08T03:41:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T03:41:33,560 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:33,560 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:33,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T03:41:33,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T03:41:33,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:33,712 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:41:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120834d8d6556e604037800d4586fc838ed1_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629292703/Put/seqid=0 2024-12-08T03:41:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742175_1351 (size=12304) 2024-12-08T03:41:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:33,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629353859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629353860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629353863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629353863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T03:41:33,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629353968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629353968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629353968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:33,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629353978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:34,126 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120834d8d6556e604037800d4586fc838ed1_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120834d8d6556e604037800d4586fc838ed1_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/838593b72738460faaf97e61e2d2eb90, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/838593b72738460faaf97e61e2d2eb90 is 175, key is test_row_0/A:col10/1733629292703/Put/seqid=0 2024-12-08T03:41:34,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742176_1352 (size=31105) 2024-12-08T03:41:34,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629354178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629354178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629354178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T03:41:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629354188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629354488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629354488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629354488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629354498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:34,530 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/838593b72738460faaf97e61e2d2eb90 2024-12-08T03:41:34,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/6aefddbd8c2c40f69f19b69c4048eff2 is 50, key is test_row_0/B:col10/1733629292703/Put/seqid=0 2024-12-08T03:41:34,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742177_1353 (size=12151) 2024-12-08T03:41:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T03:41:34,940 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/6aefddbd8c2c40f69f19b69c4048eff2 2024-12-08T03:41:34,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0977f93de8d44c7a8a2ef9971e5dcc33 is 50, key is test_row_0/C:col10/1733629292703/Put/seqid=0 2024-12-08T03:41:34,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742178_1354 (size=12151) 2024-12-08T03:41:34,963 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0977f93de8d44c7a8a2ef9971e5dcc33 2024-12-08T03:41:34,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/838593b72738460faaf97e61e2d2eb90 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90 2024-12-08T03:41:34,978 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90, entries=150, sequenceid=157, filesize=30.4 K 2024-12-08T03:41:34,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/6aefddbd8c2c40f69f19b69c4048eff2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/6aefddbd8c2c40f69f19b69c4048eff2 2024-12-08T03:41:34,987 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/6aefddbd8c2c40f69f19b69c4048eff2, entries=150, sequenceid=157, filesize=11.9 K 2024-12-08T03:41:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0977f93de8d44c7a8a2ef9971e5dcc33 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0977f93de8d44c7a8a2ef9971e5dcc33 2024-12-08T03:41:34,997 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0977f93de8d44c7a8a2ef9971e5dcc33, entries=150, sequenceid=157, filesize=11.9 K 2024-12-08T03:41:34,998 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 440aad18abead1c4be188777b931aa43 in 1286ms, sequenceid=157, compaction requested=true 2024-12-08T03:41:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-08T03:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-08T03:41:35,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:35,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:41:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:35,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-08T03:41:35,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-12-08T03:41:35,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4470 sec 2024-12-08T03:41:35,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bc0c7e4a7bb9491590965d242d04e679_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:35,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742179_1355 (size=14794) 2024-12-08T03:41:35,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629355064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629355064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629355065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629355065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629355178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629355178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629355188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629355188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629355388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629355388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629355398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629355399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:35,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bc0c7e4a7bb9491590965d242d04e679_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bc0c7e4a7bb9491590965d242d04e679_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:35,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/bda603d4cb8c486f85a689a3f3154b49, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:35,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/bda603d4cb8c486f85a689a3f3154b49 is 175, key is test_row_0/A:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:35,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742180_1356 (size=39749) 2024-12-08T03:41:35,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629355698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629355698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T03:41:35,708 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:35,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:35,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629355708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-08T03:41:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:35,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:35,711 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:35,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:35,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629355708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:35,839 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/bda603d4cb8c486f85a689a3f3154b49 2024-12-08T03:41:35,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/8d65571e2b9b448b969dd1680776faf4 is 50, key is test_row_0/B:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:35,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742181_1357 (size=12151) 2024-12-08T03:41:35,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:35,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:35,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:36,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:36,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629356208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:36,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629356208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:36,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629356218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629356228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:36,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/8d65571e2b9b448b969dd1680776faf4 2024-12-08T03:41:36,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/30975b84279748159e3cd7e7e460c93e is 50, key is test_row_0/C:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:36,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742182_1358 (size=12151) 2024-12-08T03:41:36,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:36,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:36,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/30975b84279748159e3cd7e7e460c93e 2024-12-08T03:41:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/bda603d4cb8c486f85a689a3f3154b49 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49 2024-12-08T03:41:36,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49, entries=200, sequenceid=168, filesize=38.8 K 2024-12-08T03:41:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/8d65571e2b9b448b969dd1680776faf4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/8d65571e2b9b448b969dd1680776faf4 2024-12-08T03:41:36,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/8d65571e2b9b448b969dd1680776faf4, entries=150, sequenceid=168, filesize=11.9 K 2024-12-08T03:41:36,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/30975b84279748159e3cd7e7e460c93e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/30975b84279748159e3cd7e7e460c93e 2024-12-08T03:41:36,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/30975b84279748159e3cd7e7e460c93e, entries=150, sequenceid=168, filesize=11.9 K 2024-12-08T03:41:36,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 440aad18abead1c4be188777b931aa43 in 1671ms, sequenceid=168, compaction requested=true 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:36,674 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:36,674 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/A is initiating minor compaction (all files) 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/B is initiating minor compaction (all files) 2024-12-08T03:41:36,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/A in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,675 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/B in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,675 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/c3e0914d76ac46ad9d12ab388285bd59, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/2617a73914c949d2aea1107c0b939853, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/6aefddbd8c2c40f69f19b69c4048eff2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/8d65571e2b9b448b969dd1680776faf4] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=47.4 K 2024-12-08T03:41:36,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/b87ea3605b204814a254a81d31c21d32, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=129.9 K 2024-12-08T03:41:36,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/b87ea3605b204814a254a81d31c21d32, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49] 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e0914d76ac46ad9d12ab388285bd59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629289330 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b87ea3605b204814a254a81d31c21d32, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629289330 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 2617a73914c949d2aea1107c0b939853, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733629290502 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f749ade44054e3a841ba01b359732a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733629290502 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aefddbd8c2c40f69f19b69c4048eff2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733629292703 2024-12-08T03:41:36,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 838593b72738460faaf97e61e2d2eb90, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733629292703 2024-12-08T03:41:36,676 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d65571e2b9b448b969dd1680776faf4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629293862 2024-12-08T03:41:36,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting bda603d4cb8c486f85a689a3f3154b49, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629293858 2024-12-08T03:41:36,682 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:36,682 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/467dc78a63434a5d915a30d11a9673c9 is 50, key is test_row_0/B:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:36,691 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:36,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742183_1359 (size=12527) 2024-12-08T03:41:36,698 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208f3603efe0a684a7e861f7dcd1026fba2_440aad18abead1c4be188777b931aa43 store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:36,700 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208f3603efe0a684a7e861f7dcd1026fba2_440aad18abead1c4be188777b931aa43, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:36,700 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f3603efe0a684a7e861f7dcd1026fba2_440aad18abead1c4be188777b931aa43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:36,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742184_1360 (size=4469) 2024-12-08T03:41:36,707 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#A#compaction#304 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:36,707 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/46d983e19e9741b790f5969331850a17 is 175, key is test_row_0/A:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:36,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742185_1361 (size=31481) 2024-12-08T03:41:36,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T03:41:36,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:36,779 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:41:36,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087e434960096d458392c40e45c594a1ce_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629295039/Put/seqid=0 2024-12-08T03:41:36,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742186_1362 (size=12304) 2024-12-08T03:41:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:37,099 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/467dc78a63434a5d915a30d11a9673c9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/467dc78a63434a5d915a30d11a9673c9 2024-12-08T03:41:37,106 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 440aad18abead1c4be188777b931aa43/B of 440aad18abead1c4be188777b931aa43 into 467dc78a63434a5d915a30d11a9673c9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:37,106 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:37,106 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/B, priority=12, startTime=1733629296674; duration=0sec 2024-12-08T03:41:37,106 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:37,106 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:B 2024-12-08T03:41:37,106 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:41:37,107 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:41:37,107 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/C is initiating minor compaction (all files) 2024-12-08T03:41:37,107 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/C in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:37,107 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/76590f72bf924813929f0fcdeff56181, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a2257df6432c49808182515a2e748f47, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0977f93de8d44c7a8a2ef9971e5dcc33, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/30975b84279748159e3cd7e7e460c93e] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=47.4 K 2024-12-08T03:41:37,107 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 76590f72bf924813929f0fcdeff56181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629289330 2024-12-08T03:41:37,108 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a2257df6432c49808182515a2e748f47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733629290502 2024-12-08T03:41:37,108 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0977f93de8d44c7a8a2ef9971e5dcc33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733629292703 2024-12-08T03:41:37,108 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 30975b84279748159e3cd7e7e460c93e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629293862 2024-12-08T03:41:37,114 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:37,115 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/44e177af112b493699ca5438b476811a is 50, key is test_row_0/C:col10/1733629293862/Put/seqid=0 2024-12-08T03:41:37,117 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/46d983e19e9741b790f5969331850a17 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/46d983e19e9741b790f5969331850a17 2024-12-08T03:41:37,120 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 440aad18abead1c4be188777b931aa43/A of 440aad18abead1c4be188777b931aa43 into 46d983e19e9741b790f5969331850a17(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:37,120 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:37,120 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/A, priority=12, startTime=1733629296674; duration=0sec 2024-12-08T03:41:37,120 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:37,121 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:A 2024-12-08T03:41:37,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742187_1363 (size=12527) 2024-12-08T03:41:37,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:37,192 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087e434960096d458392c40e45c594a1ce_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087e434960096d458392c40e45c594a1ce_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:37,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/76cdb934e8c74a7890cd068e6402f472, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:37,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/76cdb934e8c74a7890cd068e6402f472 is 175, key is test_row_0/A:col10/1733629295039/Put/seqid=0 2024-12-08T03:41:37,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742188_1364 (size=31105) 2024-12-08T03:41:37,197 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/76cdb934e8c74a7890cd068e6402f472 2024-12-08T03:41:37,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/7d987f47588e4053bb227cb9bf2e0f3f is 50, key is test_row_0/B:col10/1733629295039/Put/seqid=0 2024-12-08T03:41:37,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742189_1365 (size=12151) 2024-12-08T03:41:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:37,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:37,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629357235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629357238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629357238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,242 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629357248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,252 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629357349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629357349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,529 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/44e177af112b493699ca5438b476811a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/44e177af112b493699ca5438b476811a 2024-12-08T03:41:37,533 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 440aad18abead1c4be188777b931aa43/C of 440aad18abead1c4be188777b931aa43 into 44e177af112b493699ca5438b476811a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:37,533 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:37,533 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/C, priority=12, startTime=1733629296674; duration=0sec 2024-12-08T03:41:37,533 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:37,533 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:C 2024-12-08T03:41:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629357558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629357558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,609 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/7d987f47588e4053bb227cb9bf2e0f3f 2024-12-08T03:41:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/359cfd4ebcad46db8407efcb804ce39a is 50, key is test_row_0/C:col10/1733629295039/Put/seqid=0 2024-12-08T03:41:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742190_1366 (size=12151) 2024-12-08T03:41:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:37,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629357869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629357869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,018 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/359cfd4ebcad46db8407efcb804ce39a 2024-12-08T03:41:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/76cdb934e8c74a7890cd068e6402f472 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472 2024-12-08T03:41:38,025 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472, entries=150, sequenceid=194, filesize=30.4 K 2024-12-08T03:41:38,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/7d987f47588e4053bb227cb9bf2e0f3f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/7d987f47588e4053bb227cb9bf2e0f3f 2024-12-08T03:41:38,029 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/7d987f47588e4053bb227cb9bf2e0f3f, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:41:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/359cfd4ebcad46db8407efcb804ce39a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/359cfd4ebcad46db8407efcb804ce39a 2024-12-08T03:41:38,033 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/359cfd4ebcad46db8407efcb804ce39a, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:41:38,034 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 440aad18abead1c4be188777b931aa43 in 1255ms, sequenceid=194, compaction requested=false 2024-12-08T03:41:38,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:38,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:38,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-08T03:41:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-08T03:41:38,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-08T03:41:38,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3250 sec 2024-12-08T03:41:38,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3290 sec 2024-12-08T03:41:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:38,382 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:38,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:41:38,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a263799b7e764c7b9a3db48de028b9e9_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:38,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742191_1367 (size=17284) 2024-12-08T03:41:38,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629358438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629358443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629358548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629358548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629358758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:38,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629358758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:38,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:38,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a263799b7e764c7b9a3db48de028b9e9_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a263799b7e764c7b9a3db48de028b9e9_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:38,795 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/3e2723b1323c4a6385eae21c87404bff, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:38,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/3e2723b1323c4a6385eae21c87404bff is 175, key is test_row_0/A:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:38,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742192_1368 (size=48389) 2024-12-08T03:41:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629359068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629359068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:39,199 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/3e2723b1323c4a6385eae21c87404bff 2024-12-08T03:41:39,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/ee79cc54209f4bf8b99e29e6ec094b2a is 50, key is test_row_0/B:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:39,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742193_1369 (size=12151) 2024-12-08T03:41:39,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1733629359248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:39,248 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:39,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1733629359268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:39,269 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629359578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629359578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:39,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/ee79cc54209f4bf8b99e29e6ec094b2a 2024-12-08T03:41:39,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0c9a6d55a4f642e4914a1c762935deb2 is 50, key is test_row_0/C:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742194_1370 (size=12151) 2024-12-08T03:41:39,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0c9a6d55a4f642e4914a1c762935deb2 2024-12-08T03:41:39,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/3e2723b1323c4a6385eae21c87404bff as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff 2024-12-08T03:41:39,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff, entries=250, sequenceid=209, filesize=47.3 K 2024-12-08T03:41:39,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/ee79cc54209f4bf8b99e29e6ec094b2a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/ee79cc54209f4bf8b99e29e6ec094b2a 2024-12-08T03:41:39,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/ee79cc54209f4bf8b99e29e6ec094b2a, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T03:41:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/0c9a6d55a4f642e4914a1c762935deb2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0c9a6d55a4f642e4914a1c762935deb2 2024-12-08T03:41:39,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0c9a6d55a4f642e4914a1c762935deb2, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T03:41:39,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 440aad18abead1c4be188777b931aa43 in 1252ms, sequenceid=209, compaction requested=true 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:39,634 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:39,634 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:39,636 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:39,636 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:39,636 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/B is initiating minor compaction (all files) 2024-12-08T03:41:39,636 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/A is initiating minor compaction (all files) 2024-12-08T03:41:39,636 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/B in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:39,636 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/A in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:39,636 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/46d983e19e9741b790f5969331850a17, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=108.4 K 2024-12-08T03:41:39,636 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/467dc78a63434a5d915a30d11a9673c9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/7d987f47588e4053bb227cb9bf2e0f3f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/ee79cc54209f4bf8b99e29e6ec094b2a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=36.0 K 2024-12-08T03:41:39,636 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:39,636 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/46d983e19e9741b790f5969331850a17, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff] 2024-12-08T03:41:39,637 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 467dc78a63434a5d915a30d11a9673c9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629293862 2024-12-08T03:41:39,637 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d983e19e9741b790f5969331850a17, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629293862 2024-12-08T03:41:39,637 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d987f47588e4053bb227cb9bf2e0f3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629295039 2024-12-08T03:41:39,637 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76cdb934e8c74a7890cd068e6402f472, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629295039 2024-12-08T03:41:39,637 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ee79cc54209f4bf8b99e29e6ec094b2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733629297233 2024-12-08T03:41:39,637 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2723b1323c4a6385eae21c87404bff, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733629297233 2024-12-08T03:41:39,641 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:39,645 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:39,646 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/0078adae25de4bab9a4c41f4c69bf321 is 50, key is test_row_0/B:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:39,648 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208a41f633b9fa1443eb6ee454de6615d6d_440aad18abead1c4be188777b931aa43 store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:39,649 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208a41f633b9fa1443eb6ee454de6615d6d_440aad18abead1c4be188777b931aa43, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:39,650 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a41f633b9fa1443eb6ee454de6615d6d_440aad18abead1c4be188777b931aa43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:39,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742195_1371 (size=12629) 2024-12-08T03:41:39,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742196_1372 (size=4469) 2024-12-08T03:41:39,660 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#A#compaction#312 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:39,661 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/1a929407a20748b0a4ebfc81f6a216bc is 175, key is test_row_0/A:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:39,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742197_1373 (size=31583) 2024-12-08T03:41:39,670 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/1a929407a20748b0a4ebfc81f6a216bc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/1a929407a20748b0a4ebfc81f6a216bc 2024-12-08T03:41:39,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/A of 440aad18abead1c4be188777b931aa43 into 1a929407a20748b0a4ebfc81f6a216bc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:39,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:39,675 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/A, priority=13, startTime=1733629299634; duration=0sec 2024-12-08T03:41:39,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:39,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:A 2024-12-08T03:41:39,675 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:39,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:39,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/C is initiating minor compaction (all files) 2024-12-08T03:41:39,676 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/C in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:39,676 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/44e177af112b493699ca5438b476811a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/359cfd4ebcad46db8407efcb804ce39a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0c9a6d55a4f642e4914a1c762935deb2] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=36.0 K 2024-12-08T03:41:39,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e177af112b493699ca5438b476811a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733629293862 2024-12-08T03:41:39,676 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 359cfd4ebcad46db8407efcb804ce39a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629295039 2024-12-08T03:41:39,677 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c9a6d55a4f642e4914a1c762935deb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733629297233 2024-12-08T03:41:39,687 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:39,687 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/73dee5aa2fc6400dbb9177b149ae9ef8 is 50, key is test_row_0/C:col10/1733629298381/Put/seqid=0 2024-12-08T03:41:39,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742198_1374 (size=12629) 2024-12-08T03:41:39,705 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/73dee5aa2fc6400dbb9177b149ae9ef8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/73dee5aa2fc6400dbb9177b149ae9ef8 2024-12-08T03:41:39,709 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/C of 440aad18abead1c4be188777b931aa43 into 73dee5aa2fc6400dbb9177b149ae9ef8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:39,709 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:39,709 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/C, priority=13, startTime=1733629299634; duration=0sec 2024-12-08T03:41:39,709 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:39,709 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:C 2024-12-08T03:41:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T03:41:39,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:39,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-08T03:41:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T03:41:39,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:39,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:39,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T03:41:40,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T03:41:40,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:40,043 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:41:40,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120822ce34a3374640f7afa1afeb8f9d1829_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629298427/Put/seqid=0 2024-12-08T03:41:40,058 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/0078adae25de4bab9a4c41f4c69bf321 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/0078adae25de4bab9a4c41f4c69bf321 2024-12-08T03:41:40,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742199_1375 (size=12304) 2024-12-08T03:41:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:40,064 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/B of 440aad18abead1c4be188777b931aa43 into 0078adae25de4bab9a4c41f4c69bf321(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:40,064 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:40,064 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/B, priority=13, startTime=1733629299634; duration=0sec 2024-12-08T03:41:40,064 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:40,064 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:B 2024-12-08T03:41:40,066 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120822ce34a3374640f7afa1afeb8f9d1829_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120822ce34a3374640f7afa1afeb8f9d1829_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:40,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/f7c2bf2e18ad4a35887f2248f2193a2b, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:40,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/f7c2bf2e18ad4a35887f2248f2193a2b is 175, key is test_row_0/A:col10/1733629298427/Put/seqid=0 2024-12-08T03:41:40,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742200_1376 (size=31105) 2024-12-08T03:41:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T03:41:40,472 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/f7c2bf2e18ad4a35887f2248f2193a2b 2024-12-08T03:41:40,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/bc4daff374cd4d88a30787f363653043 is 50, key is test_row_0/B:col10/1733629298427/Put/seqid=0 2024-12-08T03:41:40,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742201_1377 (size=12151) 2024-12-08T03:41:40,485 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/bc4daff374cd4d88a30787f363653043 2024-12-08T03:41:40,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/a78f09a44924438fb353a50fb81a56cd is 50, key is test_row_0/C:col10/1733629298427/Put/seqid=0 2024-12-08T03:41:40,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742202_1378 (size=12151) 2024-12-08T03:41:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T03:41:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:40,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:40,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629360621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:40,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629360625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:40,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629360728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629360738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:40,896 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/a78f09a44924438fb353a50fb81a56cd 2024-12-08T03:41:40,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/f7c2bf2e18ad4a35887f2248f2193a2b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b 2024-12-08T03:41:40,903 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b, entries=150, sequenceid=235, filesize=30.4 K 2024-12-08T03:41:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/bc4daff374cd4d88a30787f363653043 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/bc4daff374cd4d88a30787f363653043 2024-12-08T03:41:40,908 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/bc4daff374cd4d88a30787f363653043, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:41:40,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/a78f09a44924438fb353a50fb81a56cd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a78f09a44924438fb353a50fb81a56cd 2024-12-08T03:41:40,912 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a78f09a44924438fb353a50fb81a56cd, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T03:41:40,913 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 440aad18abead1c4be188777b931aa43 in 870ms, sequenceid=235, compaction requested=false 2024-12-08T03:41:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-08T03:41:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-08T03:41:40,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-08T03:41:40,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-12-08T03:41:40,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0280 sec 2024-12-08T03:41:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:40,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:40,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:41:40,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ef24316d6d234659aac86fc00e2cc88e_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:40,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742203_1379 (size=14794) 2024-12-08T03:41:41,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629361003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629361011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T03:41:41,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:41,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-08T03:41:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:41,030 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:41,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:41,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:41,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629361118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629361118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:41,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629361328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629361328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:41,354 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:41,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ef24316d6d234659aac86fc00e2cc88e_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ef24316d6d234659aac86fc00e2cc88e_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:41,358 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/890fdffebc5c40239c85e0af69175c68, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:41,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/890fdffebc5c40239c85e0af69175c68 is 175, key is test_row_0/A:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:41,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742204_1380 (size=39749) 2024-12-08T03:41:41,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1733629361438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,440 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 440aad18abead1c4be188777b931aa43 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:41,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:41,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629361638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629361638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:41,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/890fdffebc5c40239c85e0af69175c68 2024-12-08T03:41:41,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/d6b729b2b3e047d1bc19e643a9495562 is 50, key is test_row_0/B:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:41,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742205_1381 (size=12151) 2024-12-08T03:41:41,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:41,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34044 deadline: 1733629362148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34074 deadline: 1733629362148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:41:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:42,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/d6b729b2b3e047d1bc19e643a9495562 2024-12-08T03:41:42,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/ff17191c6cb24b1fa9009ee83d67fac8 is 50, key is test_row_0/C:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:42,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742206_1382 (size=12151) 2024-12-08T03:41:42,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:42,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/ff17191c6cb24b1fa9009ee83d67fac8 2024-12-08T03:41:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/890fdffebc5c40239c85e0af69175c68 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68 2024-12-08T03:41:42,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68, entries=200, sequenceid=249, filesize=38.8 K 2024-12-08T03:41:42,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/d6b729b2b3e047d1bc19e643a9495562 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d6b729b2b3e047d1bc19e643a9495562 2024-12-08T03:41:42,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d6b729b2b3e047d1bc19e643a9495562, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T03:41:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/ff17191c6cb24b1fa9009ee83d67fac8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/ff17191c6cb24b1fa9009ee83d67fac8 2024-12-08T03:41:42,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/ff17191c6cb24b1fa9009ee83d67fac8, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T03:41:42,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 440aad18abead1c4be188777b931aa43 in 1650ms, sequenceid=249, compaction requested=true 2024-12-08T03:41:42,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:42,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:42,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:42,595 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:42,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:42,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:42,595 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:42,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 440aad18abead1c4be188777b931aa43:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:42,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:42,595 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/A is initiating minor compaction (all files) 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/B is initiating minor compaction (all files) 2024-12-08T03:41:42,596 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/B in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,596 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/A in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,596 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/1a929407a20748b0a4ebfc81f6a216bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=100.0 K 2024-12-08T03:41:42,596 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/0078adae25de4bab9a4c41f4c69bf321, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/bc4daff374cd4d88a30787f363653043, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d6b729b2b3e047d1bc19e643a9495562] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=36.1 K 2024-12-08T03:41:42,596 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/1a929407a20748b0a4ebfc81f6a216bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68] 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0078adae25de4bab9a4c41f4c69bf321, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733629297233 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a929407a20748b0a4ebfc81f6a216bc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733629297233 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4daff374cd4d88a30787f363653043, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629298427 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c2bf2e18ad4a35887f2248f2193a2b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629298427 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b729b2b3e047d1bc19e643a9495562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733629300609 2024-12-08T03:41:42,596 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890fdffebc5c40239c85e0af69175c68, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733629300609 2024-12-08T03:41:42,600 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:42,601 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087157628d4e14404f94bcf26fe455bd76_440aad18abead1c4be188777b931aa43 store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:42,602 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#B#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:42,602 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/602b2099f18243749877ac953d7dec80 is 50, key is test_row_0/B:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:42,603 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087157628d4e14404f94bcf26fe455bd76_440aad18abead1c4be188777b931aa43, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:42,603 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087157628d4e14404f94bcf26fe455bd76_440aad18abead1c4be188777b931aa43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742207_1383 (size=12731) 2024-12-08T03:41:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742208_1384 (size=4469) 2024-12-08T03:41:42,613 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#A#compaction#321 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:42,613 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/3750475d674a4578ab2157001a9e0aa5 is 175, key is test_row_0/A:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:42,619 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/602b2099f18243749877ac953d7dec80 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/602b2099f18243749877ac953d7dec80 2024-12-08T03:41:42,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742209_1385 (size=31685) 2024-12-08T03:41:42,625 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/B of 440aad18abead1c4be188777b931aa43 into 602b2099f18243749877ac953d7dec80(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:42,625 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:42,625 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/B, priority=13, startTime=1733629302595; duration=0sec 2024-12-08T03:41:42,625 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:42,625 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:B 2024-12-08T03:41:42,625 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/3750475d674a4578ab2157001a9e0aa5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3750475d674a4578ab2157001a9e0aa5 2024-12-08T03:41:42,626 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:42,626 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:42,626 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 440aad18abead1c4be188777b931aa43/C is initiating minor compaction (all files) 2024-12-08T03:41:42,626 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 440aad18abead1c4be188777b931aa43/C in TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,626 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/73dee5aa2fc6400dbb9177b149ae9ef8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a78f09a44924438fb353a50fb81a56cd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/ff17191c6cb24b1fa9009ee83d67fac8] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp, totalSize=36.1 K 2024-12-08T03:41:42,626 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 73dee5aa2fc6400dbb9177b149ae9ef8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733629297233 2024-12-08T03:41:42,627 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a78f09a44924438fb353a50fb81a56cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733629298427 2024-12-08T03:41:42,627 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ff17191c6cb24b1fa9009ee83d67fac8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733629300609 2024-12-08T03:41:42,629 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/A of 440aad18abead1c4be188777b931aa43 into 3750475d674a4578ab2157001a9e0aa5(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:42,629 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:42,629 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/A, priority=13, startTime=1733629302595; duration=0sec 2024-12-08T03:41:42,630 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:42,630 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:A 2024-12-08T03:41:42,633 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 440aad18abead1c4be188777b931aa43#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:42,633 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/bd9be215461a4ae4845e839ffe499ee7 is 50, key is test_row_0/C:col10/1733629300942/Put/seqid=0 2024-12-08T03:41:42,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742210_1386 (size=12731) 2024-12-08T03:41:42,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T03:41:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:42,709 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:41:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086bdab1193059480bb079ddb51880fc84_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629301010/Put/seqid=0 2024-12-08T03:41:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742211_1387 (size=12454) 2024-12-08T03:41:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:42,725 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086bdab1193059480bb079ddb51880fc84_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086bdab1193059480bb079ddb51880fc84_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:42,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d90bbd488192413d9fcfdd219bb8246b, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:42,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d90bbd488192413d9fcfdd219bb8246b is 175, key is test_row_0/A:col10/1733629301010/Put/seqid=0 2024-12-08T03:41:42,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742212_1388 (size=31255) 2024-12-08T03:41:43,072 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-12-08T03:41:43,073 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,073 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,073 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,073 INFO [Thread-1454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1454. 2024-12-08T03:41:43,073 DEBUG [Thread-1454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,073 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,073 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,074 INFO [Thread-1458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1458. 2024-12-08T03:41:43,075 DEBUG [Thread-1458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,075 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,075 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,076 INFO [Thread-1456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1456. 2024-12-08T03:41:43,076 DEBUG [Thread-1456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,076 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,076 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,077 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-12-08T03:41:43,077 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,077 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,077 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,080 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/bd9be215461a4ae4845e839ffe499ee7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/bd9be215461a4ae4845e839ffe499ee7 2024-12-08T03:41:43,084 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 440aad18abead1c4be188777b931aa43/C of 440aad18abead1c4be188777b931aa43 into bd9be215461a4ae4845e839ffe499ee7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:43,084 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:43,084 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., storeName=440aad18abead1c4be188777b931aa43/C, priority=13, startTime=1733629302595; duration=0sec 2024-12-08T03:41:43,084 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:43,084 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 440aad18abead1c4be188777b931aa43:C 2024-12-08T03:41:43,134 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d90bbd488192413d9fcfdd219bb8246b 2024-12-08T03:41:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/acd6a8ca71344399bb652a0b355d6b44 is 50, key is test_row_0/B:col10/1733629301010/Put/seqid=0 2024-12-08T03:41:43,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742213_1389 (size=12301) 2024-12-08T03:41:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. as already flushing 2024-12-08T03:41:43,160 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-12-08T03:41:43,160 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,160 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,160 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,170 INFO [Thread-1449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1449. 2024-12-08T03:41:43,170 DEBUG [Thread-1449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,170 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,170 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:43,260 INFO [Thread-1445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1445. 2024-12-08T03:41:43,260 DEBUG [Thread-1445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,260 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,260 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,292 INFO [Thread-1447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1447. 2024-12-08T03:41:43,292 DEBUG [Thread-1447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:43,292 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,292 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:43,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:43,544 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/acd6a8ca71344399bb652a0b355d6b44 2024-12-08T03:41:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/32f0e4ca01c5440dbff73f11ba7cc0e3 is 50, key is test_row_0/C:col10/1733629301010/Put/seqid=0 2024-12-08T03:41:43,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742214_1390 (size=12301) 2024-12-08T03:41:43,966 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/32f0e4ca01c5440dbff73f11ba7cc0e3 2024-12-08T03:41:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/d90bbd488192413d9fcfdd219bb8246b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d90bbd488192413d9fcfdd219bb8246b 2024-12-08T03:41:43,981 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d90bbd488192413d9fcfdd219bb8246b, entries=150, sequenceid=274, filesize=30.5 K 2024-12-08T03:41:43,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/acd6a8ca71344399bb652a0b355d6b44 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/acd6a8ca71344399bb652a0b355d6b44 2024-12-08T03:41:43,987 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/acd6a8ca71344399bb652a0b355d6b44, entries=150, sequenceid=274, filesize=12.0 K 2024-12-08T03:41:43,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/32f0e4ca01c5440dbff73f11ba7cc0e3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/32f0e4ca01c5440dbff73f11ba7cc0e3 2024-12-08T03:41:43,993 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/32f0e4ca01c5440dbff73f11ba7cc0e3, entries=150, sequenceid=274, filesize=12.0 K 2024-12-08T03:41:43,994 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=26.84 KB/27480 for 440aad18abead1c4be188777b931aa43 in 1285ms, sequenceid=274, compaction requested=false 2024-12-08T03:41:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 440aad18abead1c4be188777b931aa43: 2024-12-08T03:41:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-08T03:41:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-08T03:41:43,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-08T03:41:43,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9640 sec 2024-12-08T03:41:43,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.9690 sec 2024-12-08T03:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T03:41:45,199 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:51,463 INFO [Thread-1443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1443. 2024-12-08T03:41:51,463 DEBUG [Thread-1443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:41:51,463 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:51,463 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:51,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1924 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5765 rows 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1918 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5749 rows 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1933 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5787 rows 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1923 2024-12-08T03:41:51,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5769 rows 2024-12-08T03:41:51,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1934 2024-12-08T03:41:51,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5801 rows 2024-12-08T03:41:51,465 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:41:51,465 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:41:51,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:51,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:51,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:41:51,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T03:41:51,473 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629311473"}]},"ts":"1733629311473"} 2024-12-08T03:41:51,475 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:41:51,475 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:41:51,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:41:51,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, UNASSIGN}] 2024-12-08T03:41:51,477 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, UNASSIGN 2024-12-08T03:41:51,478 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:51,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, UNASSIGN because future has completed 2024-12-08T03:41:51,479 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:41:51,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T03:41:51,632 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:51,632 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:41:51,633 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 440aad18abead1c4be188777b931aa43, disabling compactions & flushes 2024-12-08T03:41:51,633 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:51,633 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:51,633 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. after waiting 0 ms 2024-12-08T03:41:51,633 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:51,633 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 440aad18abead1c4be188777b931aa43 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-08T03:41:51,647 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120861db7203925b454d9770284a506c027c_440aad18abead1c4be188777b931aa43 is 50, key is test_row_0/A:col10/1733629303169/Put/seqid=0 2024-12-08T03:41:51,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742215_1391 (size=12454) 2024-12-08T03:41:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T03:41:52,051 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:52,059 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120861db7203925b454d9770284a506c027c_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861db7203925b454d9770284a506c027c_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:52,060 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/a1b25a4e05244775897121498cf915b7, store: [table=TestAcidGuarantees family=A region=440aad18abead1c4be188777b931aa43] 2024-12-08T03:41:52,061 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/a1b25a4e05244775897121498cf915b7 is 175, key is test_row_0/A:col10/1733629303169/Put/seqid=0 2024-12-08T03:41:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742216_1392 (size=31255) 2024-12-08T03:41:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T03:41:52,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:41:52,466 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/a1b25a4e05244775897121498cf915b7 2024-12-08T03:41:52,480 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/255e6eb04f704e7dbdae216dbc2457e3 is 50, key is test_row_0/B:col10/1733629303169/Put/seqid=0 2024-12-08T03:41:52,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742217_1393 (size=12301) 2024-12-08T03:41:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T03:41:52,885 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/255e6eb04f704e7dbdae216dbc2457e3 2024-12-08T03:41:52,899 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/5579e479c798489781c95761dae65542 is 50, key is test_row_0/C:col10/1733629303169/Put/seqid=0 2024-12-08T03:41:52,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742218_1394 (size=12301) 2024-12-08T03:41:53,306 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/5579e479c798489781c95761dae65542 2024-12-08T03:41:53,355 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/A/a1b25a4e05244775897121498cf915b7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/a1b25a4e05244775897121498cf915b7 2024-12-08T03:41:53,361 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/a1b25a4e05244775897121498cf915b7, entries=150, sequenceid=283, filesize=30.5 K 2024-12-08T03:41:53,362 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/B/255e6eb04f704e7dbdae216dbc2457e3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/255e6eb04f704e7dbdae216dbc2457e3 2024-12-08T03:41:53,368 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/255e6eb04f704e7dbdae216dbc2457e3, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T03:41:53,369 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/.tmp/C/5579e479c798489781c95761dae65542 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/5579e479c798489781c95761dae65542 2024-12-08T03:41:53,373 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/5579e479c798489781c95761dae65542, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T03:41:53,374 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 440aad18abead1c4be188777b931aa43 in 1741ms, sequenceid=283, compaction requested=true 2024-12-08T03:41:53,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e3825f8769324ffcb51d072f0e778dc5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/b87ea3605b204814a254a81d31c21d32, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/46d983e19e9741b790f5969331850a17, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/1a929407a20748b0a4ebfc81f6a216bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68] to archive 2024-12-08T03:41:53,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:41:53,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e1ebb83081d64ca694126211e5aab1c6 2024-12-08T03:41:53,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d178ad80a7de49c59d69196a95f482ed 2024-12-08T03:41:53,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e3825f8769324ffcb51d072f0e778dc5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/e3825f8769324ffcb51d072f0e778dc5 2024-12-08T03:41:53,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/c02e650619b34b509600f861a56a3bc4 2024-12-08T03:41:53,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/ccd07b9d906a425e8f536841c68a89d7 2024-12-08T03:41:53,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/67f99be048be44e8b17835779aa46bbb 2024-12-08T03:41:53,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/94485b4eec9249c39b0dfa4283dd5100 2024-12-08T03:41:53,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/b87ea3605b204814a254a81d31c21d32 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/b87ea3605b204814a254a81d31c21d32 2024-12-08T03:41:53,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/5f749ade44054e3a841ba01b359732a4 2024-12-08T03:41:53,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/838593b72738460faaf97e61e2d2eb90 2024-12-08T03:41:53,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/bda603d4cb8c486f85a689a3f3154b49 2024-12-08T03:41:53,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/46d983e19e9741b790f5969331850a17 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/46d983e19e9741b790f5969331850a17 2024-12-08T03:41:53,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/76cdb934e8c74a7890cd068e6402f472 2024-12-08T03:41:53,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3e2723b1323c4a6385eae21c87404bff 2024-12-08T03:41:53,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/1a929407a20748b0a4ebfc81f6a216bc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/1a929407a20748b0a4ebfc81f6a216bc 2024-12-08T03:41:53,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/f7c2bf2e18ad4a35887f2248f2193a2b 2024-12-08T03:41:53,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/890fdffebc5c40239c85e0af69175c68 2024-12-08T03:41:53,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/fec7a6862c0847d59f24a96e67de3e7a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d93ebd93682e4d2ebad20006d591454b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/482362db451b4991b82fe1558a09c064, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/9003850dae5249a0a4581f615cf29241, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/eab88cb88c4a41ed88de5df67934a344, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/e506fb4d64694fc79cc6f61fbacf4cf2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/c3e0914d76ac46ad9d12ab388285bd59, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/f0917e2c97594cc9a088bb9526582d5b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/2617a73914c949d2aea1107c0b939853, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/6aefddbd8c2c40f69f19b69c4048eff2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/467dc78a63434a5d915a30d11a9673c9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/8d65571e2b9b448b969dd1680776faf4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/7d987f47588e4053bb227cb9bf2e0f3f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/0078adae25de4bab9a4c41f4c69bf321, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/ee79cc54209f4bf8b99e29e6ec094b2a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/bc4daff374cd4d88a30787f363653043, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d6b729b2b3e047d1bc19e643a9495562] to archive 2024-12-08T03:41:53,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:41:53,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/fec7a6862c0847d59f24a96e67de3e7a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/fec7a6862c0847d59f24a96e67de3e7a 2024-12-08T03:41:53,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d93ebd93682e4d2ebad20006d591454b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d93ebd93682e4d2ebad20006d591454b 2024-12-08T03:41:53,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/482362db451b4991b82fe1558a09c064 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/482362db451b4991b82fe1558a09c064 2024-12-08T03:41:53,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/9003850dae5249a0a4581f615cf29241 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/9003850dae5249a0a4581f615cf29241 2024-12-08T03:41:53,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/eab88cb88c4a41ed88de5df67934a344 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/eab88cb88c4a41ed88de5df67934a344 2024-12-08T03:41:53,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/e506fb4d64694fc79cc6f61fbacf4cf2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/e506fb4d64694fc79cc6f61fbacf4cf2 2024-12-08T03:41:53,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/c3e0914d76ac46ad9d12ab388285bd59 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/c3e0914d76ac46ad9d12ab388285bd59 2024-12-08T03:41:53,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/f0917e2c97594cc9a088bb9526582d5b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/f0917e2c97594cc9a088bb9526582d5b 2024-12-08T03:41:53,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/2617a73914c949d2aea1107c0b939853 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/2617a73914c949d2aea1107c0b939853 2024-12-08T03:41:53,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/6aefddbd8c2c40f69f19b69c4048eff2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/6aefddbd8c2c40f69f19b69c4048eff2 2024-12-08T03:41:53,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/467dc78a63434a5d915a30d11a9673c9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/467dc78a63434a5d915a30d11a9673c9 2024-12-08T03:41:53,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/8d65571e2b9b448b969dd1680776faf4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/8d65571e2b9b448b969dd1680776faf4 2024-12-08T03:41:53,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/7d987f47588e4053bb227cb9bf2e0f3f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/7d987f47588e4053bb227cb9bf2e0f3f 2024-12-08T03:41:53,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/0078adae25de4bab9a4c41f4c69bf321 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/0078adae25de4bab9a4c41f4c69bf321 2024-12-08T03:41:53,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/ee79cc54209f4bf8b99e29e6ec094b2a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/ee79cc54209f4bf8b99e29e6ec094b2a 2024-12-08T03:41:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/bc4daff374cd4d88a30787f363653043 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/bc4daff374cd4d88a30787f363653043 2024-12-08T03:41:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d6b729b2b3e047d1bc19e643a9495562 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/d6b729b2b3e047d1bc19e643a9495562 2024-12-08T03:41:53,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/d559749b3ea842beb23d9c59b220c5a7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0bd945ccec294b26abce4ddc8d49e2e8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/91c48cbe563d43c5b3449b8bfb62adc5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/52848e5b4afa4a79ad9826054d4ff772, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/27f88c75d0364bce9882c212c082b602, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/be50aa76951a45f8b9564372f5c58b49, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/76590f72bf924813929f0fcdeff56181, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/2896425d25a545ec9018a89495951b07, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a2257df6432c49808182515a2e748f47, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0977f93de8d44c7a8a2ef9971e5dcc33, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/44e177af112b493699ca5438b476811a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/30975b84279748159e3cd7e7e460c93e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/359cfd4ebcad46db8407efcb804ce39a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/73dee5aa2fc6400dbb9177b149ae9ef8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0c9a6d55a4f642e4914a1c762935deb2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a78f09a44924438fb353a50fb81a56cd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/ff17191c6cb24b1fa9009ee83d67fac8] to archive 2024-12-08T03:41:53,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:41:53,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/d559749b3ea842beb23d9c59b220c5a7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/d559749b3ea842beb23d9c59b220c5a7 2024-12-08T03:41:53,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0bd945ccec294b26abce4ddc8d49e2e8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0bd945ccec294b26abce4ddc8d49e2e8 2024-12-08T03:41:53,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/91c48cbe563d43c5b3449b8bfb62adc5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/91c48cbe563d43c5b3449b8bfb62adc5 2024-12-08T03:41:53,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/52848e5b4afa4a79ad9826054d4ff772 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/52848e5b4afa4a79ad9826054d4ff772 2024-12-08T03:41:53,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/27f88c75d0364bce9882c212c082b602 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/27f88c75d0364bce9882c212c082b602 2024-12-08T03:41:53,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/be50aa76951a45f8b9564372f5c58b49 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/be50aa76951a45f8b9564372f5c58b49 2024-12-08T03:41:53,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/76590f72bf924813929f0fcdeff56181 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/76590f72bf924813929f0fcdeff56181 2024-12-08T03:41:53,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/2896425d25a545ec9018a89495951b07 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/2896425d25a545ec9018a89495951b07 2024-12-08T03:41:53,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a2257df6432c49808182515a2e748f47 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a2257df6432c49808182515a2e748f47 2024-12-08T03:41:53,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0977f93de8d44c7a8a2ef9971e5dcc33 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0977f93de8d44c7a8a2ef9971e5dcc33 2024-12-08T03:41:53,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/44e177af112b493699ca5438b476811a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/44e177af112b493699ca5438b476811a 2024-12-08T03:41:53,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/30975b84279748159e3cd7e7e460c93e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/30975b84279748159e3cd7e7e460c93e 2024-12-08T03:41:53,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/359cfd4ebcad46db8407efcb804ce39a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/359cfd4ebcad46db8407efcb804ce39a 2024-12-08T03:41:53,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/73dee5aa2fc6400dbb9177b149ae9ef8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/73dee5aa2fc6400dbb9177b149ae9ef8 2024-12-08T03:41:53,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0c9a6d55a4f642e4914a1c762935deb2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/0c9a6d55a4f642e4914a1c762935deb2 2024-12-08T03:41:53,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a78f09a44924438fb353a50fb81a56cd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/a78f09a44924438fb353a50fb81a56cd 2024-12-08T03:41:53,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/ff17191c6cb24b1fa9009ee83d67fac8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/ff17191c6cb24b1fa9009ee83d67fac8 2024-12-08T03:41:53,429 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/recovered.edits/286.seqid, newMaxSeqId=286, maxSeqId=4 2024-12-08T03:41:53,429 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. 2024-12-08T03:41:53,429 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 440aad18abead1c4be188777b931aa43: Waiting for close lock at 1733629311633Running coprocessor pre-close hooks at 1733629311633Disabling compacts and flushes for region at 1733629311633Disabling writes for close at 1733629311633Obtaining lock to block concurrent updates at 1733629311633Preparing flush snapshotting stores in 440aad18abead1c4be188777b931aa43 at 1733629311633Finished memstore snapshotting TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733629311634 (+1 ms)Flushing stores of TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43. at 1733629311635 (+1 ms)Flushing 440aad18abead1c4be188777b931aa43/A: creating writer at 1733629311635Flushing mob file 440aad18abead1c4be188777b931aa43/A: appending metadata at 1733629311646 (+11 ms)Flushing mob file 440aad18abead1c4be188777b931aa43/A: closing flushed file at 1733629311646Flushing 440aad18abead1c4be188777b931aa43/A: appending metadata at 1733629312060 (+414 ms)Flushing 440aad18abead1c4be188777b931aa43/A: closing flushed file at 1733629312060Flushing 440aad18abead1c4be188777b931aa43/B: creating writer at 1733629312475 (+415 ms)Flushing 440aad18abead1c4be188777b931aa43/B: appending metadata at 1733629312479 (+4 ms)Flushing 440aad18abead1c4be188777b931aa43/B: closing flushed file at 1733629312479Flushing 440aad18abead1c4be188777b931aa43/C: creating writer at 1733629312894 (+415 ms)Flushing 440aad18abead1c4be188777b931aa43/C: appending metadata at 1733629312898 (+4 ms)Flushing 440aad18abead1c4be188777b931aa43/C: closing flushed file at 1733629312898Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53941f91: reopening flushed file at 1733629313354 (+456 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bdab7bc: reopening flushed file at 1733629313361 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@171f1154: reopening flushed file at 1733629313368 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 440aad18abead1c4be188777b931aa43 in 1741ms, sequenceid=283, compaction requested=true at 1733629313374 (+6 ms)Writing region close event to WAL at 1733629313427 (+53 ms)Running coprocessor post-close hooks at 1733629313429 (+2 ms)Closed at 1733629313429 2024-12-08T03:41:53,431 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=440aad18abead1c4be188777b931aa43, regionState=CLOSED 2024-12-08T03:41:53,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:41:53,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-08T03:41:53,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 440aad18abead1c4be188777b931aa43, server=32df4b4bb439,42629,1733629194165 in 1.9540 sec 2024-12-08T03:41:53,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-08T03:41:53,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=440aad18abead1c4be188777b931aa43, UNASSIGN in 1.9570 sec 2024-12-08T03:41:53,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-08T03:41:53,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9610 sec 2024-12-08T03:41:53,438 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629313438"}]},"ts":"1733629313438"} 2024-12-08T03:41:53,439 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:41:53,439 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:41:53,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9710 sec 2024-12-08T03:41:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T03:41:53,619 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:53,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:41:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,624 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T03:41:53,626 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,632 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/recovered.edits] 2024-12-08T03:41:53,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3750475d674a4578ab2157001a9e0aa5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/3750475d674a4578ab2157001a9e0aa5 2024-12-08T03:41:53,635 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/a1b25a4e05244775897121498cf915b7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/a1b25a4e05244775897121498cf915b7 2024-12-08T03:41:53,636 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d90bbd488192413d9fcfdd219bb8246b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/A/d90bbd488192413d9fcfdd219bb8246b 2024-12-08T03:41:53,638 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/255e6eb04f704e7dbdae216dbc2457e3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/255e6eb04f704e7dbdae216dbc2457e3 2024-12-08T03:41:53,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/602b2099f18243749877ac953d7dec80 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/602b2099f18243749877ac953d7dec80 2024-12-08T03:41:53,640 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/acd6a8ca71344399bb652a0b355d6b44 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/B/acd6a8ca71344399bb652a0b355d6b44 2024-12-08T03:41:53,641 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/32f0e4ca01c5440dbff73f11ba7cc0e3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/32f0e4ca01c5440dbff73f11ba7cc0e3 2024-12-08T03:41:53,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/5579e479c798489781c95761dae65542 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/5579e479c798489781c95761dae65542 2024-12-08T03:41:53,643 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/bd9be215461a4ae4845e839ffe499ee7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/C/bd9be215461a4ae4845e839ffe499ee7 2024-12-08T03:41:53,645 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/recovered.edits/286.seqid to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43/recovered.edits/286.seqid 2024-12-08T03:41:53,645 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,645 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:41:53,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:41:53,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T03:41:53,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080be25b456d98448997de03373f8d9950_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080be25b456d98448997de03373f8d9950_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208115a4d011f514cfaa401ea91100f8d42_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208115a4d011f514cfaa401ea91100f8d42_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120822ce34a3374640f7afa1afeb8f9d1829_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120822ce34a3374640f7afa1afeb8f9d1829_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120834d8d6556e604037800d4586fc838ed1_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120834d8d6556e604037800d4586fc838ed1_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,652 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861db7203925b454d9770284a506c027c_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861db7203925b454d9770284a506c027c_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086bdab1193059480bb079ddb51880fc84_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086bdab1193059480bb079ddb51880fc84_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087e434960096d458392c40e45c594a1ce_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087e434960096d458392c40e45c594a1ce_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,655 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089a2450c5a0ba4f798e2aae63c77ace16_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089a2450c5a0ba4f798e2aae63c77ace16_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a263799b7e764c7b9a3db48de028b9e9_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a263799b7e764c7b9a3db48de028b9e9_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,657 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b9ae548debc04374ab9afae2f00b31e2_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b9ae548debc04374ab9afae2f00b31e2_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,657 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208baa85e1c565c45779d9af29ddb3135fe_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208baa85e1c565c45779d9af29ddb3135fe_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,658 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bc0c7e4a7bb9491590965d242d04e679_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bc0c7e4a7bb9491590965d242d04e679_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,659 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d6d381e90a964e1b87e26f52f2d4476a_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d6d381e90a964e1b87e26f52f2d4476a_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,659 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3db6f562ef14260b858cc9c61d8aec9_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3db6f562ef14260b858cc9c61d8aec9_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,660 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ef24316d6d234659aac86fc00e2cc88e_440aad18abead1c4be188777b931aa43 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ef24316d6d234659aac86fc00e2cc88e_440aad18abead1c4be188777b931aa43 2024-12-08T03:41:53,660 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:41:53,662 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,663 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:41:53,665 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:41:53,666 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,666 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:41:53,666 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733629313666"}]},"ts":"9223372036854775807"} 2024-12-08T03:41:53,668 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:41:53,668 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 440aad18abead1c4be188777b931aa43, NAME => 'TestAcidGuarantees,,1733629279902.440aad18abead1c4be188777b931aa43.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:41:53,668 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:41:53,668 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733629313668"}]},"ts":"9223372036854775807"} 2024-12-08T03:41:53,669 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:41:53,670 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-08T03:41:53,681 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T03:41:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T03:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:41:53,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:53,753 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 241), OpenFileDescriptor=445 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=17151 (was 17179) 2024-12-08T03:41:53,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=17151 2024-12-08T03:41:53,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:41:53,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:41:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:41:53,765 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:41:53,765 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:53,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-12-08T03:41:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T03:41:53,766 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:41:53,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742219_1395 (size=959) 2024-12-08T03:41:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T03:41:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T03:41:54,177 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:41:54,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742220_1396 (size=53) 2024-12-08T03:41:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T03:41:54,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:41:54,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5a84826978f2928496b58ad318fa3531, disabling compactions & flushes 2024-12-08T03:41:54,588 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. after waiting 0 ms 2024-12-08T03:41:54,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,588 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a84826978f2928496b58ad318fa3531: Waiting for close lock at 1733629314588Disabling compacts and flushes for region at 1733629314588Disabling writes for close at 1733629314588Writing region close event to WAL at 1733629314588Closed at 1733629314588 2024-12-08T03:41:54,591 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:41:54,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733629314591"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733629314591"}]},"ts":"1733629314591"} 2024-12-08T03:41:54,596 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:41:54,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:41:54,598 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629314598"}]},"ts":"1733629314598"} 2024-12-08T03:41:54,601 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:41:54,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, ASSIGN}] 2024-12-08T03:41:54,603 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, ASSIGN 2024-12-08T03:41:54,604 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, ASSIGN; state=OFFLINE, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=false 2024-12-08T03:41:54,755 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=5a84826978f2928496b58ad318fa3531, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:54,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, ASSIGN because future has completed 2024-12-08T03:41:54,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:41:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T03:41:54,918 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,919 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:41:54,920 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,920 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:41:54,921 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,921 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,922 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,924 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a84826978f2928496b58ad318fa3531 columnFamilyName A 2024-12-08T03:41:54,924 DEBUG [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:54,925 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] regionserver.HStore(327): Store=5a84826978f2928496b58ad318fa3531/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:54,925 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,926 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a84826978f2928496b58ad318fa3531 columnFamilyName B 2024-12-08T03:41:54,927 DEBUG [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:54,927 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] regionserver.HStore(327): Store=5a84826978f2928496b58ad318fa3531/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:54,927 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,929 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a84826978f2928496b58ad318fa3531 columnFamilyName C 2024-12-08T03:41:54,929 DEBUG [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:41:54,929 INFO [StoreOpener-5a84826978f2928496b58ad318fa3531-1 {}] regionserver.HStore(327): Store=5a84826978f2928496b58ad318fa3531/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:41:54,929 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,930 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,931 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,932 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,932 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,932 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:41:54,933 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,934 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:41:54,935 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 5a84826978f2928496b58ad318fa3531; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65792211, jitterRate=-0.019619658589363098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:41:54,935 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:54,935 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 5a84826978f2928496b58ad318fa3531: Running coprocessor pre-open hook at 1733629314921Writing region info on filesystem at 1733629314921Initializing all the Stores at 1733629314922 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629314922Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629314922Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629314922Cleaning up temporary data from old regions at 1733629314932 (+10 ms)Running coprocessor post-open hooks at 1733629314935 (+3 ms)Region opened successfully at 1733629314935 2024-12-08T03:41:54,936 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., pid=116, masterSystemTime=1733629314911 2024-12-08T03:41:54,937 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,938 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:54,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=5a84826978f2928496b58ad318fa3531, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:54,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:41:54,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-08T03:41:54,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 in 181 msec 2024-12-08T03:41:54,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-08T03:41:54,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, ASSIGN in 339 msec 2024-12-08T03:41:54,943 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:41:54,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629314943"}]},"ts":"1733629314943"} 2024-12-08T03:41:54,944 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:41:54,944 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:41:54,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-12-08T03:41:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T03:41:55,910 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:55,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,915 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cd3ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6abff712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,921 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:41:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:41:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38890a0f 2024-12-08T03:41:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:41:55,926 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:41:55,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,928 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1361d7f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f281bba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2371eeca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,933 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2704f2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,935 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c9e6f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d65a8dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,939 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d480dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,941 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2feb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,943 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38da0f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,944 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461c4bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,946 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1682995a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,948 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43843d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,950 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66235cdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417ed292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129ea183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1779fecc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,956 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478c260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,958 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52737c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:41:55,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:41:55,959 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:41:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:41:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:41:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e364158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:41:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:41:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:41:55,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:41:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158a9445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:41:55,961 DEBUG [Thread-1810 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,961 DEBUG [Thread-1814 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,961 DEBUG [Thread-1801 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,961 DEBUG [Thread-1799 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,962 DEBUG [Thread-1805 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,962 DEBUG [Thread-1807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,962 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,962 DEBUG [Thread-1818 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,963 DEBUG [Thread-1816 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-08T03:41:55,964 DEBUG [Thread-1803 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,965 DEBUG [Thread-1812 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:41:55,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T03:41:55,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,966 DEBUG [Thread-1812 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,966 DEBUG [Thread-1812 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:41:55,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:55,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:55,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:55,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:41:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2] 2024-12-08T03:41:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:55,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:55,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:41:56,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/78f5b69b39cb48759bc075baf0384e34 is 50, key is test_row_0/A:col10/1733629315980/Put/seqid=0 2024-12-08T03:41:56,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742221_1397 (size=14341) 2024-12-08T03:41:56,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/78f5b69b39cb48759bc075baf0384e34 2024-12-08T03:41:56,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629376010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629376011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629376012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629376019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629376021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/bd1adec3771d4473bb859463e1587116 is 50, key is test_row_0/B:col10/1733629315980/Put/seqid=0 2024-12-08T03:41:56,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742222_1398 (size=12001) 2024-12-08T03:41:56,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/bd1adec3771d4473bb859463e1587116 2024-12-08T03:41:56,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8e16871089c549309a99d623fab796f1 is 50, key is test_row_0/C:col10/1733629315980/Put/seqid=0 2024-12-08T03:41:56,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742223_1399 (size=12001) 2024-12-08T03:41:56,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8e16871089c549309a99d623fab796f1 2024-12-08T03:41:56,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/78f5b69b39cb48759bc075baf0384e34 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/78f5b69b39cb48759bc075baf0384e34 2024-12-08T03:41:56,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/78f5b69b39cb48759bc075baf0384e34, entries=200, sequenceid=14, filesize=14.0 K 2024-12-08T03:41:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T03:41:56,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/bd1adec3771d4473bb859463e1587116 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/bd1adec3771d4473bb859463e1587116 2024-12-08T03:41:56,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/bd1adec3771d4473bb859463e1587116, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T03:41:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8e16871089c549309a99d623fab796f1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e16871089c549309a99d623fab796f1 2024-12-08T03:41:56,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e16871089c549309a99d623fab796f1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T03:41:56,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5a84826978f2928496b58ad318fa3531 in 99ms, sequenceid=14, compaction requested=false 2024-12-08T03:41:56,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:56,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T03:41:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:56,124 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:41:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/65320e541e994fa3b588652a92f0d513 is 50, key is test_row_0/A:col10/1733629316017/Put/seqid=0 2024-12-08T03:41:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:56,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:56,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742224_1400 (size=12001) 2024-12-08T03:41:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629376138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629376139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629376141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629376142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629376145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629376248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629376258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629376258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629376259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T03:41:56,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629376348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629376458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629376468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629376468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629376468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,536 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/65320e541e994fa3b588652a92f0d513 2024-12-08T03:41:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/15cac7c39acc453cb62fbbde6a6548fa is 50, key is test_row_0/B:col10/1733629316017/Put/seqid=0 2024-12-08T03:41:56,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742225_1401 (size=12001) 2024-12-08T03:41:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T03:41:56,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629376658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629376768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629376778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629376779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629376779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:56,944 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/15cac7c39acc453cb62fbbde6a6548fa 2024-12-08T03:41:56,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/2731df018b574ab181749ad5720d23e3 is 50, key is test_row_0/C:col10/1733629316017/Put/seqid=0 2024-12-08T03:41:56,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742226_1402 (size=12001) 2024-12-08T03:41:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T03:41:57,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629377168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629377278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:57,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629377288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629377288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629377288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:57,353 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/2731df018b574ab181749ad5720d23e3 2024-12-08T03:41:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/65320e541e994fa3b588652a92f0d513 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/65320e541e994fa3b588652a92f0d513 2024-12-08T03:41:57,359 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/65320e541e994fa3b588652a92f0d513, entries=150, sequenceid=38, filesize=11.7 K 2024-12-08T03:41:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/15cac7c39acc453cb62fbbde6a6548fa as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/15cac7c39acc453cb62fbbde6a6548fa 2024-12-08T03:41:57,362 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/15cac7c39acc453cb62fbbde6a6548fa, entries=150, sequenceid=38, filesize=11.7 K 2024-12-08T03:41:57,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/2731df018b574ab181749ad5720d23e3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/2731df018b574ab181749ad5720d23e3 2024-12-08T03:41:57,366 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/2731df018b574ab181749ad5720d23e3, entries=150, sequenceid=38, filesize=11.7 K 2024-12-08T03:41:57,367 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5a84826978f2928496b58ad318fa3531 in 1243ms, sequenceid=38, compaction requested=false 2024-12-08T03:41:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-08T03:41:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-08T03:41:57,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-08T03:41:57,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-12-08T03:41:57,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4090 sec 2024-12-08T03:41:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T03:41:58,108 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:41:58,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:41:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-08T03:41:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T03:41:58,110 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:41:58,110 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:41:58,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:41:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:58,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:41:58,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/7ff7cb355fe542c7918c7b1a162ef102 is 50, key is test_row_0/A:col10/1733629316143/Put/seqid=0 2024-12-08T03:41:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742227_1403 (size=14341) 2024-12-08T03:41:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T03:41:58,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629378274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629378298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,302 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629378299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629378299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629378299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,305 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,305 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,306 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629378388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T03:41:58,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/7ff7cb355fe542c7918c7b1a162ef102 2024-12-08T03:41:58,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/5a94743bab68457f8352d73babf29b6e is 50, key is test_row_0/B:col10/1733629316143/Put/seqid=0 2024-12-08T03:41:58,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742228_1404 (size=12001) 2024-12-08T03:41:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629378608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T03:41:58,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:58,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:58,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629378918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:58,949 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:41:59,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/5a94743bab68457f8352d73babf29b6e 2024-12-08T03:41:59,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/a22833440b6b4886835a828079dacaf0 is 50, key is test_row_0/C:col10/1733629316143/Put/seqid=0 2024-12-08T03:41:59,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742229_1405 (size=12001) 2024-12-08T03:41:59,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/a22833440b6b4886835a828079dacaf0 2024-12-08T03:41:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/7ff7cb355fe542c7918c7b1a162ef102 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/7ff7cb355fe542c7918c7b1a162ef102 2024-12-08T03:41:59,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/7ff7cb355fe542c7918c7b1a162ef102, entries=200, sequenceid=51, filesize=14.0 K 2024-12-08T03:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/5a94743bab68457f8352d73babf29b6e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a94743bab68457f8352d73babf29b6e 2024-12-08T03:41:59,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a94743bab68457f8352d73babf29b6e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T03:41:59,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/a22833440b6b4886835a828079dacaf0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/a22833440b6b4886835a828079dacaf0 2024-12-08T03:41:59,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:59,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/a22833440b6b4886835a828079dacaf0, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T03:41:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:41:59,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5a84826978f2928496b58ad318fa3531 in 844ms, sequenceid=51, compaction requested=true 2024-12-08T03:41:59,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:41:59,027 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:59,027 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:41:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:41:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:59,027 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:59,027 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:59,027 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:41:59,027 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:41:59,028 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,028 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,028 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/78f5b69b39cb48759bc075baf0384e34, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/65320e541e994fa3b588652a92f0d513, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/7ff7cb355fe542c7918c7b1a162ef102] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=39.7 K 2024-12-08T03:41:59,028 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/bd1adec3771d4473bb859463e1587116, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/15cac7c39acc453cb62fbbde6a6548fa, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a94743bab68457f8352d73babf29b6e] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=35.2 K 2024-12-08T03:41:59,028 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bd1adec3771d4473bb859463e1587116, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733629315977 2024-12-08T03:41:59,028 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f5b69b39cb48759bc075baf0384e34, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733629315973 2024-12-08T03:41:59,028 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65320e541e994fa3b588652a92f0d513, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733629316007 2024-12-08T03:41:59,028 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 15cac7c39acc453cb62fbbde6a6548fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733629316007 2024-12-08T03:41:59,028 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ff7cb355fe542c7918c7b1a162ef102, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629316132 2024-12-08T03:41:59,028 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a94743bab68457f8352d73babf29b6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629316141 2024-12-08T03:41:59,041 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:59,041 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:59,041 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/47b9d694bc7b4ac3a9e507b6e3e2eb09 is 50, key is test_row_0/B:col10/1733629316143/Put/seqid=0 2024-12-08T03:41:59,041 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/97227e312cc44ad6a67816a80bd5ea79 is 50, key is test_row_0/A:col10/1733629316143/Put/seqid=0 2024-12-08T03:41:59,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742230_1406 (size=12104) 2024-12-08T03:41:59,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742231_1407 (size=12104) 2024-12-08T03:41:59,091 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/47b9d694bc7b4ac3a9e507b6e3e2eb09 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47b9d694bc7b4ac3a9e507b6e3e2eb09 2024-12-08T03:41:59,096 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into 47b9d694bc7b4ac3a9e507b6e3e2eb09(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:59,096 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:59,096 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629319027; duration=0sec 2024-12-08T03:41:59,096 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:41:59,096 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:41:59,096 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:41:59,097 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:41:59,097 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:41:59,097 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,097 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e16871089c549309a99d623fab796f1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/2731df018b574ab181749ad5720d23e3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/a22833440b6b4886835a828079dacaf0] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=35.2 K 2024-12-08T03:41:59,097 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e16871089c549309a99d623fab796f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733629315977 2024-12-08T03:41:59,098 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 2731df018b574ab181749ad5720d23e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733629316007 2024-12-08T03:41:59,098 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a22833440b6b4886835a828079dacaf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629316141 2024-12-08T03:41:59,106 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:41:59,106 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/7351ce19086244b487f7a23c80ccb000 is 50, key is test_row_0/C:col10/1733629316143/Put/seqid=0 2024-12-08T03:41:59,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742232_1408 (size=12104) 2024-12-08T03:41:59,131 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/7351ce19086244b487f7a23c80ccb000 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/7351ce19086244b487f7a23c80ccb000 2024-12-08T03:41:59,140 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into 7351ce19086244b487f7a23c80ccb000(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:59,140 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:59,140 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629319027; duration=0sec 2024-12-08T03:41:59,140 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:59,140 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:41:59,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T03:41:59,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,177 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:41:59,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ec314efbaf5b4017850a807a9c794ecb is 50, key is test_row_0/A:col10/1733629318273/Put/seqid=0 2024-12-08T03:41:59,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742233_1409 (size=12001) 2024-12-08T03:41:59,186 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ec314efbaf5b4017850a807a9c794ecb 2024-12-08T03:41:59,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/4973a6dc8af34f7a8409561bfa5fb6ae is 50, key is test_row_0/B:col10/1733629318273/Put/seqid=0 2024-12-08T03:41:59,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742234_1410 (size=12001) 2024-12-08T03:41:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T03:41:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:59,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:41:59,462 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/97227e312cc44ad6a67816a80bd5ea79 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/97227e312cc44ad6a67816a80bd5ea79 2024-12-08T03:41:59,466 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into 97227e312cc44ad6a67816a80bd5ea79(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:41:59,466 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:59,466 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629319026; duration=0sec 2024-12-08T03:41:59,466 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:41:59,466 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:41:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629379484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:59,595 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/4973a6dc8af34f7a8409561bfa5fb6ae 2024-12-08T03:41:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/d2c376176f0a4002b32c0f4ac86420a1 is 50, key is test_row_0/C:col10/1733629318273/Put/seqid=0 2024-12-08T03:41:59,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629379598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:41:59,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742235_1411 (size=12001) 2024-12-08T03:41:59,607 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/d2c376176f0a4002b32c0f4ac86420a1 2024-12-08T03:41:59,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ec314efbaf5b4017850a807a9c794ecb as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ec314efbaf5b4017850a807a9c794ecb 2024-12-08T03:41:59,618 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ec314efbaf5b4017850a807a9c794ecb, entries=150, sequenceid=76, filesize=11.7 K 2024-12-08T03:41:59,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/4973a6dc8af34f7a8409561bfa5fb6ae as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/4973a6dc8af34f7a8409561bfa5fb6ae 2024-12-08T03:41:59,624 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/4973a6dc8af34f7a8409561bfa5fb6ae, entries=150, sequenceid=76, filesize=11.7 K 2024-12-08T03:41:59,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/d2c376176f0a4002b32c0f4ac86420a1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d2c376176f0a4002b32c0f4ac86420a1 2024-12-08T03:41:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d2c376176f0a4002b32c0f4ac86420a1, entries=150, sequenceid=76, filesize=11.7 K 2024-12-08T03:41:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5a84826978f2928496b58ad318fa3531 in 452ms, sequenceid=76, compaction requested=false 2024-12-08T03:41:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:41:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:41:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-08T03:41:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-08T03:41:59,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-08T03:41:59,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5210 sec 2024-12-08T03:41:59,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5250 sec 2024-12-08T03:41:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:41:59,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:41:59,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-08T03:41:59,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/264eb332f32249c8ab4583084051e8d5 is 50, key is test_row_0/A:col10/1733629319468/Put/seqid=0 2024-12-08T03:41:59,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742236_1412 (size=14341) 2024-12-08T03:41:59,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:41:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629379903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:41:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:41:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629380018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/264eb332f32249c8ab4583084051e8d5 2024-12-08T03:42:00,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/47f09bc6f42a44e3ac265524d22106bf is 50, key is test_row_0/B:col10/1733629319468/Put/seqid=0 2024-12-08T03:42:00,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742237_1413 (size=12001) 2024-12-08T03:42:00,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629380228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T03:42:00,258 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:00,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-08T03:42:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:00,260 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:00,260 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:00,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629380308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,313 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629380318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,321 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629380328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629380328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,334 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,335 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:00,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629380538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:00,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:00,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/47f09bc6f42a44e3ac265524d22106bf 2024-12-08T03:42:00,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/0992d90e27d341c5ba90e933940bf978 is 50, key is test_row_0/C:col10/1733629319468/Put/seqid=0 2024-12-08T03:42:00,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742238_1414 (size=12001) 2024-12-08T03:42:00,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:00,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:01,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/0992d90e27d341c5ba90e933940bf978 2024-12-08T03:42:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/264eb332f32249c8ab4583084051e8d5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/264eb332f32249c8ab4583084051e8d5 2024-12-08T03:42:01,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/264eb332f32249c8ab4583084051e8d5, entries=200, sequenceid=91, filesize=14.0 K 2024-12-08T03:42:01,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/47f09bc6f42a44e3ac265524d22106bf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47f09bc6f42a44e3ac265524d22106bf 2024-12-08T03:42:01,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47f09bc6f42a44e3ac265524d22106bf, entries=150, sequenceid=91, filesize=11.7 K 2024-12-08T03:42:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/0992d90e27d341c5ba90e933940bf978 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0992d90e27d341c5ba90e933940bf978 2024-12-08T03:42:01,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0992d90e27d341c5ba90e933940bf978, entries=150, sequenceid=91, filesize=11.7 K 2024-12-08T03:42:01,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5a84826978f2928496b58ad318fa3531 in 1240ms, sequenceid=91, compaction requested=true 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:01,051 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:01,051 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:01,051 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:01,052 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:01,052 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,052 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47b9d694bc7b4ac3a9e507b6e3e2eb09, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/4973a6dc8af34f7a8409561bfa5fb6ae, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47f09bc6f42a44e3ac265524d22106bf] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=35.3 K 2024-12-08T03:42:01,052 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:01,052 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:01,052 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,052 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/97227e312cc44ad6a67816a80bd5ea79, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ec314efbaf5b4017850a807a9c794ecb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/264eb332f32249c8ab4583084051e8d5] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=37.5 K 2024-12-08T03:42:01,052 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b9d694bc7b4ac3a9e507b6e3e2eb09, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629316141 2024-12-08T03:42:01,052 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97227e312cc44ad6a67816a80bd5ea79, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629316141 2024-12-08T03:42:01,052 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec314efbaf5b4017850a807a9c794ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733629318263 2024-12-08T03:42:01,053 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 4973a6dc8af34f7a8409561bfa5fb6ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733629318263 2024-12-08T03:42:01,053 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f09bc6f42a44e3ac265524d22106bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629319468 2024-12-08T03:42:01,053 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264eb332f32249c8ab4583084051e8d5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629319468 2024-12-08T03:42:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:01,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:01,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-08T03:42:01,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/f72ec2f1fdff436594c4e78775274688 is 50, key is test_row_0/A:col10/1733629319895/Put/seqid=0 2024-12-08T03:42:01,060 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:01,060 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:01,060 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/b1ecdd083c2343dca4a5f06dea6a922f is 50, key is test_row_0/B:col10/1733629319468/Put/seqid=0 2024-12-08T03:42:01,061 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/137ceda3bac54bba93a2e2f9dd840a28 is 50, key is test_row_0/A:col10/1733629319468/Put/seqid=0 2024-12-08T03:42:01,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742239_1415 (size=12207) 2024-12-08T03:42:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742241_1417 (size=14341) 2024-12-08T03:42:01,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742240_1416 (size=12207) 2024-12-08T03:42:01,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/f72ec2f1fdff436594c4e78775274688 2024-12-08T03:42:01,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1959c344238e4ee1be0cadc213359bb5 is 50, key is test_row_0/B:col10/1733629319895/Put/seqid=0 2024-12-08T03:42:01,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742242_1418 (size=12001) 2024-12-08T03:42:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629381108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:01,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629381219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:01,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:01,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629381428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:01,469 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/b1ecdd083c2343dca4a5f06dea6a922f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/b1ecdd083c2343dca4a5f06dea6a922f 2024-12-08T03:42:01,474 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/137ceda3bac54bba93a2e2f9dd840a28 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/137ceda3bac54bba93a2e2f9dd840a28 2024-12-08T03:42:01,474 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into b1ecdd083c2343dca4a5f06dea6a922f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:01,474 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:01,474 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629321051; duration=0sec 2024-12-08T03:42:01,474 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:01,474 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:01,474 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:01,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1959c344238e4ee1be0cadc213359bb5 2024-12-08T03:42:01,476 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:01,476 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:01,476 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,476 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/7351ce19086244b487f7a23c80ccb000, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d2c376176f0a4002b32c0f4ac86420a1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0992d90e27d341c5ba90e933940bf978] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=35.3 K 2024-12-08T03:42:01,477 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7351ce19086244b487f7a23c80ccb000, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733629316141 2024-12-08T03:42:01,477 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c376176f0a4002b32c0f4ac86420a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733629318263 2024-12-08T03:42:01,477 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0992d90e27d341c5ba90e933940bf978, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629319468 2024-12-08T03:42:01,478 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into 137ceda3bac54bba93a2e2f9dd840a28(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:01,478 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:01,478 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629321051; duration=0sec 2024-12-08T03:42:01,478 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:01,478 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:01,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:01,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/03f396017dc24d4e941018033cf614f5 is 50, key is test_row_0/C:col10/1733629319895/Put/seqid=0 2024-12-08T03:42:01,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742243_1419 (size=12001) 2024-12-08T03:42:01,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/03f396017dc24d4e941018033cf614f5 2024-12-08T03:42:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/f72ec2f1fdff436594c4e78775274688 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f72ec2f1fdff436594c4e78775274688 2024-12-08T03:42:01,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f72ec2f1fdff436594c4e78775274688, entries=200, sequenceid=114, filesize=14.0 K 2024-12-08T03:42:01,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1959c344238e4ee1be0cadc213359bb5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1959c344238e4ee1be0cadc213359bb5 2024-12-08T03:42:01,492 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:01,493 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/67d844494da44981a825724c78a8a8cf is 50, key is test_row_0/C:col10/1733629319468/Put/seqid=0 2024-12-08T03:42:01,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1959c344238e4ee1be0cadc213359bb5, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T03:42:01,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/03f396017dc24d4e941018033cf614f5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/03f396017dc24d4e941018033cf614f5 2024-12-08T03:42:01,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742244_1420 (size=12207) 2024-12-08T03:42:01,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/03f396017dc24d4e941018033cf614f5, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T03:42:01,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5a84826978f2928496b58ad318fa3531 in 446ms, sequenceid=114, compaction requested=false 2024-12-08T03:42:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:01,505 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/67d844494da44981a825724c78a8a8cf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/67d844494da44981a825724c78a8a8cf 2024-12-08T03:42:01,510 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into 67d844494da44981a825724c78a8a8cf(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:01,510 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:01,510 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629321051; duration=0sec 2024-12-08T03:42:01,510 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:01,510 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:01,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T03:42:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:01,637 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:42:01,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/31de2c68045d4bdaa930006c56636ea4 is 50, key is test_row_1/A:col10/1733629321106/Put/seqid=0 2024-12-08T03:42:01,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742245_1421 (size=9657) 2024-12-08T03:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629381838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:01,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629381948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:02,045 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/31de2c68045d4bdaa930006c56636ea4 2024-12-08T03:42:02,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1bca9da79a0b4b9a857504b7bcac6ad9 is 50, key is test_row_1/B:col10/1733629321106/Put/seqid=0 2024-12-08T03:42:02,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742246_1422 (size=9657) 2024-12-08T03:42:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629382158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:02,460 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1bca9da79a0b4b9a857504b7bcac6ad9 2024-12-08T03:42:02,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/1721ea9bea274ab0bed9d1b388c9ceec is 50, key is test_row_1/C:col10/1733629321106/Put/seqid=0 2024-12-08T03:42:02,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742247_1423 (size=9657) 2024-12-08T03:42:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629382468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:02,870 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/1721ea9bea274ab0bed9d1b388c9ceec 2024-12-08T03:42:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/31de2c68045d4bdaa930006c56636ea4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/31de2c68045d4bdaa930006c56636ea4 2024-12-08T03:42:02,876 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/31de2c68045d4bdaa930006c56636ea4, entries=100, sequenceid=130, filesize=9.4 K 2024-12-08T03:42:02,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1bca9da79a0b4b9a857504b7bcac6ad9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1bca9da79a0b4b9a857504b7bcac6ad9 2024-12-08T03:42:02,880 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1bca9da79a0b4b9a857504b7bcac6ad9, entries=100, sequenceid=130, filesize=9.4 K 2024-12-08T03:42:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/1721ea9bea274ab0bed9d1b388c9ceec as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1721ea9bea274ab0bed9d1b388c9ceec 2024-12-08T03:42:02,883 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1721ea9bea274ab0bed9d1b388c9ceec, entries=100, sequenceid=130, filesize=9.4 K 2024-12-08T03:42:02,884 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5a84826978f2928496b58ad318fa3531 in 1247ms, sequenceid=130, compaction requested=true 2024-12-08T03:42:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-08T03:42:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-08T03:42:02,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-08T03:42:02,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6250 sec 2024-12-08T03:42:02,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6290 sec 2024-12-08T03:42:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:02,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:42:02,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/d0774698219448a9963576790e0b64a2 is 50, key is test_row_0/A:col10/1733629321836/Put/seqid=0 2024-12-08T03:42:02,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742248_1424 (size=14541) 2024-12-08T03:42:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629383028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:03,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629383138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629383348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:03,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/d0774698219448a9963576790e0b64a2 2024-12-08T03:42:03,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/26a081a94fd04cc6a42071639af47bc9 is 50, key is test_row_0/B:col10/1733629321836/Put/seqid=0 2024-12-08T03:42:03,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742249_1425 (size=12151) 2024-12-08T03:42:03,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629383658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:03,681 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T03:42:03,681 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T03:42:03,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/26a081a94fd04cc6a42071639af47bc9 2024-12-08T03:42:03,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/bbb80ebdb70149e3adb5fe0567f099f0 is 50, key is test_row_0/C:col10/1733629321836/Put/seqid=0 2024-12-08T03:42:03,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742250_1426 (size=12151) 2024-12-08T03:42:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629384168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/bbb80ebdb70149e3adb5fe0567f099f0 2024-12-08T03:42:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/d0774698219448a9963576790e0b64a2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d0774698219448a9963576790e0b64a2 2024-12-08T03:42:04,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d0774698219448a9963576790e0b64a2, entries=200, sequenceid=154, filesize=14.2 K 2024-12-08T03:42:04,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/26a081a94fd04cc6a42071639af47bc9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/26a081a94fd04cc6a42071639af47bc9 2024-12-08T03:42:04,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/26a081a94fd04cc6a42071639af47bc9, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T03:42:04,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/bbb80ebdb70149e3adb5fe0567f099f0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/bbb80ebdb70149e3adb5fe0567f099f0 2024-12-08T03:42:04,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/bbb80ebdb70149e3adb5fe0567f099f0, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T03:42:04,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5a84826978f2928496b58ad318fa3531 in 1287ms, sequenceid=154, compaction requested=true 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:04,271 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:04,271 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:04,272 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50746 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:42:04,272 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:04,272 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,272 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/137ceda3bac54bba93a2e2f9dd840a28, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f72ec2f1fdff436594c4e78775274688, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/31de2c68045d4bdaa930006c56636ea4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d0774698219448a9963576790e0b64a2] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=49.6 K 2024-12-08T03:42:04,272 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:42:04,272 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:04,272 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,272 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/b1ecdd083c2343dca4a5f06dea6a922f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1959c344238e4ee1be0cadc213359bb5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1bca9da79a0b4b9a857504b7bcac6ad9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/26a081a94fd04cc6a42071639af47bc9] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=44.9 K 2024-12-08T03:42:04,272 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137ceda3bac54bba93a2e2f9dd840a28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629319468 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ecdd083c2343dca4a5f06dea6a922f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629319468 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f72ec2f1fdff436594c4e78775274688, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733629319876 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1959c344238e4ee1be0cadc213359bb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733629319876 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31de2c68045d4bdaa930006c56636ea4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733629321100 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bca9da79a0b4b9a857504b7bcac6ad9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733629321100 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0774698219448a9963576790e0b64a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629321796 2024-12-08T03:42:04,273 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a081a94fd04cc6a42071639af47bc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629321796 2024-12-08T03:42:04,280 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:04,281 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1684fa2f63fc4fb89f48aafb3d4bc17e is 50, key is test_row_0/B:col10/1733629321836/Put/seqid=0 2024-12-08T03:42:04,281 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:04,281 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/5c2a076d645a47569712260dbb7b20c3 is 50, key is test_row_0/A:col10/1733629321836/Put/seqid=0 2024-12-08T03:42:04,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742251_1427 (size=12493) 2024-12-08T03:42:04,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742252_1428 (size=12493) 2024-12-08T03:42:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:04,343 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:04,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:42:04,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/bee7d76e9be14dcc887f195e1574e71a is 50, key is test_row_0/A:col10/1733629323027/Put/seqid=0 2024-12-08T03:42:04,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742253_1429 (size=14541) 2024-12-08T03:42:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629384393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629384395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629384402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629384403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T03:42:04,429 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:04,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-08T03:42:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:04,430 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:04,431 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:04,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629384508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629384508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629384518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629384518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:04,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,692 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1684fa2f63fc4fb89f48aafb3d4bc17e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1684fa2f63fc4fb89f48aafb3d4bc17e 2024-12-08T03:42:04,702 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/5c2a076d645a47569712260dbb7b20c3 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/5c2a076d645a47569712260dbb7b20c3 2024-12-08T03:42:04,702 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into 1684fa2f63fc4fb89f48aafb3d4bc17e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:04,702 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:04,702 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=12, startTime=1733629324271; duration=0sec 2024-12-08T03:42:04,702 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:04,702 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:04,703 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:42:04,703 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:42:04,703 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:04,704 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,704 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/67d844494da44981a825724c78a8a8cf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/03f396017dc24d4e941018033cf614f5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1721ea9bea274ab0bed9d1b388c9ceec, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/bbb80ebdb70149e3adb5fe0567f099f0] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=44.9 K 2024-12-08T03:42:04,704 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d844494da44981a825724c78a8a8cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733629319468 2024-12-08T03:42:04,705 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f396017dc24d4e941018033cf614f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733629319876 2024-12-08T03:42:04,705 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1721ea9bea274ab0bed9d1b388c9ceec, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733629321100 2024-12-08T03:42:04,705 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into 5c2a076d645a47569712260dbb7b20c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:04,705 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb80ebdb70149e3adb5fe0567f099f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629321796 2024-12-08T03:42:04,706 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:04,706 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=12, startTime=1733629324271; duration=0sec 2024-12-08T03:42:04,706 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:04,706 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:04,710 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#363 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:04,711 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/e2b204a61a084239b337941b781d2998 is 50, key is test_row_0/C:col10/1733629321836/Put/seqid=0 2024-12-08T03:42:04,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742254_1430 (size=12493) 2024-12-08T03:42:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629384718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629384718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:04,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629384728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629384728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:04,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/bee7d76e9be14dcc887f195e1574e71a 2024-12-08T03:42:04,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a9ea692c016f4415ace6ec7edb8425cf is 50, key is test_row_0/B:col10/1733629323027/Put/seqid=0 2024-12-08T03:42:04,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742255_1431 (size=12151) 2024-12-08T03:42:04,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a9ea692c016f4415ace6ec7edb8425cf 2024-12-08T03:42:04,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/9b166d3a33c24f70b905b9c777c47fd9 is 50, key is test_row_0/C:col10/1733629323027/Put/seqid=0 2024-12-08T03:42:04,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742256_1432 (size=12151) 2024-12-08T03:42:04,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:04,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629385048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:05,117 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/e2b204a61a084239b337941b781d2998 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e2b204a61a084239b337941b781d2998 2024-12-08T03:42:05,120 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into e2b204a61a084239b337941b781d2998(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:05,120 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:05,120 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=12, startTime=1733629324271; duration=0sec 2024-12-08T03:42:05,120 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:05,120 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:05,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/9b166d3a33c24f70b905b9c777c47fd9 2024-12-08T03:42:05,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/bee7d76e9be14dcc887f195e1574e71a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bee7d76e9be14dcc887f195e1574e71a 2024-12-08T03:42:05,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bee7d76e9be14dcc887f195e1574e71a, entries=200, sequenceid=167, filesize=14.2 K 2024-12-08T03:42:05,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a9ea692c016f4415ace6ec7edb8425cf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a9ea692c016f4415ace6ec7edb8425cf 2024-12-08T03:42:05,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a9ea692c016f4415ace6ec7edb8425cf, entries=150, sequenceid=167, filesize=11.9 K 2024-12-08T03:42:05,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/9b166d3a33c24f70b905b9c777c47fd9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9b166d3a33c24f70b905b9c777c47fd9 2024-12-08T03:42:05,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9b166d3a33c24f70b905b9c777c47fd9, entries=150, sequenceid=167, filesize=11.9 K 2024-12-08T03:42:05,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5a84826978f2928496b58ad318fa3531 in 839ms, sequenceid=167, compaction requested=false 2024-12-08T03:42:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:05,185 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:05,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:42:05,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/b4bc8360dd914f03ae128fc290e98188 is 50, key is test_row_0/A:col10/1733629324394/Put/seqid=0 2024-12-08T03:42:05,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742257_1433 (size=14541) 2024-12-08T03:42:05,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629385234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629385348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629385548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629385548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:05,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/b4bc8360dd914f03ae128fc290e98188 2024-12-08T03:42:05,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/358eb653f9114db3a6a2ce85fdc6b185 is 50, key is test_row_0/B:col10/1733629324394/Put/seqid=0 2024-12-08T03:42:05,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742258_1434 (size=12151) 2024-12-08T03:42:05,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629385868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:05,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:05,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:05,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:05,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/358eb653f9114db3a6a2ce85fdc6b185 2024-12-08T03:42:06,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8e3effecc734497081a007c66f5aa1e6 is 50, key is test_row_0/C:col10/1733629324394/Put/seqid=0 2024-12-08T03:42:06,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742259_1435 (size=12151) 2024-12-08T03:42:06,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629386378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:06,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8e3effecc734497081a007c66f5aa1e6 2024-12-08T03:42:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/b4bc8360dd914f03ae128fc290e98188 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/b4bc8360dd914f03ae128fc290e98188 2024-12-08T03:42:06,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/b4bc8360dd914f03ae128fc290e98188, entries=200, sequenceid=194, filesize=14.2 K 2024-12-08T03:42:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/358eb653f9114db3a6a2ce85fdc6b185 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/358eb653f9114db3a6a2ce85fdc6b185 2024-12-08T03:42:06,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/358eb653f9114db3a6a2ce85fdc6b185, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:42:06,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8e3effecc734497081a007c66f5aa1e6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e3effecc734497081a007c66f5aa1e6 2024-12-08T03:42:06,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e3effecc734497081a007c66f5aa1e6, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T03:42:06,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5a84826978f2928496b58ad318fa3531 in 1254ms, sequenceid=194, compaction requested=true 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:06,439 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:06,439 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:06,447 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:06,447 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:06,447 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,447 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/5c2a076d645a47569712260dbb7b20c3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bee7d76e9be14dcc887f195e1574e71a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/b4bc8360dd914f03ae128fc290e98188] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=40.6 K 2024-12-08T03:42:06,447 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:06,448 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,448 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1684fa2f63fc4fb89f48aafb3d4bc17e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a9ea692c016f4415ace6ec7edb8425cf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/358eb653f9114db3a6a2ce85fdc6b185] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=35.9 K 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c2a076d645a47569712260dbb7b20c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629321796 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1684fa2f63fc4fb89f48aafb3d4bc17e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629321796 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting bee7d76e9be14dcc887f195e1574e71a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733629323006 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ea692c016f4415ace6ec7edb8425cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733629323026 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4bc8360dd914f03ae128fc290e98188, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629324392 2024-12-08T03:42:06,448 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 358eb653f9114db3a6a2ce85fdc6b185, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629324394 2024-12-08T03:42:06,454 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:06,454 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:06,454 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ad620fbaf354438eb3b100f90c7f8af8 is 50, key is test_row_0/A:col10/1733629324394/Put/seqid=0 2024-12-08T03:42:06,454 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/387a7fa1739f40079fa44c87c56428d1 is 50, key is test_row_0/B:col10/1733629324394/Put/seqid=0 2024-12-08T03:42:06,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742260_1436 (size=12595) 2024-12-08T03:42:06,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742261_1437 (size=12595) 2024-12-08T03:42:06,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T03:42:06,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,567 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:42:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/173a447346074e45b77534c1a336944a is 50, key is test_row_0/A:col10/1733629325232/Put/seqid=0 2024-12-08T03:42:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:06,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742262_1438 (size=12151) 2024-12-08T03:42:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629386602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629386603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629386605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629386610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629386718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629386718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629386728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629386728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,868 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ad620fbaf354438eb3b100f90c7f8af8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ad620fbaf354438eb3b100f90c7f8af8 2024-12-08T03:42:06,868 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/387a7fa1739f40079fa44c87c56428d1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/387a7fa1739f40079fa44c87c56428d1 2024-12-08T03:42:06,872 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into ad620fbaf354438eb3b100f90c7f8af8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:06,872 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into 387a7fa1739f40079fa44c87c56428d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:06,872 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629326439; duration=0sec 2024-12-08T03:42:06,872 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629326439; duration=0sec 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:06,872 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:06,873 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:06,873 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:06,873 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:06,873 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e2b204a61a084239b337941b781d2998, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9b166d3a33c24f70b905b9c777c47fd9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e3effecc734497081a007c66f5aa1e6] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=35.9 K 2024-12-08T03:42:06,873 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b204a61a084239b337941b781d2998, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733629321796 2024-12-08T03:42:06,873 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b166d3a33c24f70b905b9c777c47fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733629323026 2024-12-08T03:42:06,874 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e3effecc734497081a007c66f5aa1e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629324394 2024-12-08T03:42:06,878 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:06,879 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/ed2faae1c7484ba69bf465870b7e97c8 is 50, key is test_row_0/C:col10/1733629324394/Put/seqid=0 2024-12-08T03:42:06,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742263_1439 (size=12595) 2024-12-08T03:42:06,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629386938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629386938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:06,974 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/173a447346074e45b77534c1a336944a 2024-12-08T03:42:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1d43245e31b54b11a151a22c41b401e2 is 50, key is test_row_0/B:col10/1733629325232/Put/seqid=0 2024-12-08T03:42:06,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742264_1440 (size=12151) 2024-12-08T03:42:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629387239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629387239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629387248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629387248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,285 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/ed2faae1c7484ba69bf465870b7e97c8 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ed2faae1c7484ba69bf465870b7e97c8 2024-12-08T03:42:07,288 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into ed2faae1c7484ba69bf465870b7e97c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:07,288 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:07,288 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629326439; duration=0sec 2024-12-08T03:42:07,288 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:07,289 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:07,383 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1d43245e31b54b11a151a22c41b401e2 2024-12-08T03:42:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/040a538e465845d4858d1d5fd723e2ed is 50, key is test_row_0/C:col10/1733629325232/Put/seqid=0 2024-12-08T03:42:07,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742265_1441 (size=12151) 2024-12-08T03:42:07,391 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/040a538e465845d4858d1d5fd723e2ed 2024-12-08T03:42:07,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/173a447346074e45b77534c1a336944a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/173a447346074e45b77534c1a336944a 2024-12-08T03:42:07,397 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/173a447346074e45b77534c1a336944a, entries=150, sequenceid=206, filesize=11.9 K 2024-12-08T03:42:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/1d43245e31b54b11a151a22c41b401e2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1d43245e31b54b11a151a22c41b401e2 2024-12-08T03:42:07,400 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1d43245e31b54b11a151a22c41b401e2, entries=150, sequenceid=206, filesize=11.9 K 2024-12-08T03:42:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/040a538e465845d4858d1d5fd723e2ed as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/040a538e465845d4858d1d5fd723e2ed 2024-12-08T03:42:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629387398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,403 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,405 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/040a538e465845d4858d1d5fd723e2ed, entries=150, sequenceid=206, filesize=11.9 K 2024-12-08T03:42:07,406 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5a84826978f2928496b58ad318fa3531 in 839ms, sequenceid=206, compaction requested=false 2024-12-08T03:42:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-08T03:42:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-08T03:42:07,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-08T03:42:07,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9770 sec 2024-12-08T03:42:07,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.9800 sec 2024-12-08T03:42:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:07,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-08T03:42:07,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/1d234bd5d00b4da29ebdb9bd6fd7c789 is 50, key is test_row_0/A:col10/1733629327751/Put/seqid=0 2024-12-08T03:42:07,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742266_1442 (size=14541) 2024-12-08T03:42:07,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/1d234bd5d00b4da29ebdb9bd6fd7c789 2024-12-08T03:42:07,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/642c77c6a21c47aa8933d043803ff5c9 is 50, key is test_row_0/B:col10/1733629327751/Put/seqid=0 2024-12-08T03:42:07,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742267_1443 (size=12151) 2024-12-08T03:42:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629387763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629387764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629387767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629387769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629387878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629387878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629387879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629387879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629388088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629388088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629388089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629388089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/642c77c6a21c47aa8933d043803ff5c9 2024-12-08T03:42:08,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d is 50, key is test_row_0/C:col10/1733629327751/Put/seqid=0 2024-12-08T03:42:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742268_1444 (size=12151) 2024-12-08T03:42:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629388398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629388398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629388398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629388398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d 2024-12-08T03:42:08,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/1d234bd5d00b4da29ebdb9bd6fd7c789 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/1d234bd5d00b4da29ebdb9bd6fd7c789 2024-12-08T03:42:08,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/1d234bd5d00b4da29ebdb9bd6fd7c789, entries=200, sequenceid=234, filesize=14.2 K 2024-12-08T03:42:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/642c77c6a21c47aa8933d043803ff5c9 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/642c77c6a21c47aa8933d043803ff5c9 2024-12-08T03:42:08,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/642c77c6a21c47aa8933d043803ff5c9, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T03:42:08,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d 2024-12-08T03:42:08,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T03:42:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T03:42:08,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5a84826978f2928496b58ad318fa3531 in 846ms, sequenceid=234, compaction requested=true 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:08,598 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:08,598 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:08,598 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:08,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-08T03:42:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T03:42:08,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:08,601 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:08,601 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:08,601 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:08,601 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:08,601 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:08,601 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:08,601 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/387a7fa1739f40079fa44c87c56428d1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1d43245e31b54b11a151a22c41b401e2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/642c77c6a21c47aa8933d043803ff5c9] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.0 K 2024-12-08T03:42:08,601 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ad620fbaf354438eb3b100f90c7f8af8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/173a447346074e45b77534c1a336944a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/1d234bd5d00b4da29ebdb9bd6fd7c789] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=38.4 K 2024-12-08T03:42:08,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:08,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:08,602 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad620fbaf354438eb3b100f90c7f8af8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629324394 2024-12-08T03:42:08,602 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 387a7fa1739f40079fa44c87c56428d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629324394 2024-12-08T03:42:08,602 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d43245e31b54b11a151a22c41b401e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733629325199 2024-12-08T03:42:08,602 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 173a447346074e45b77534c1a336944a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733629325199 2024-12-08T03:42:08,603 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 642c77c6a21c47aa8933d043803ff5c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733629326602 2024-12-08T03:42:08,603 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d234bd5d00b4da29ebdb9bd6fd7c789, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733629326601 2024-12-08T03:42:08,609 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:08,610 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:08,610 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/2c727b080a7645c681d99594eb6c6150 is 50, key is test_row_0/B:col10/1733629327751/Put/seqid=0 2024-12-08T03:42:08,610 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/20add23d751843a7844d3ca8a53299cc is 50, key is test_row_0/A:col10/1733629327751/Put/seqid=0 2024-12-08T03:42:08,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742269_1445 (size=12697) 2024-12-08T03:42:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742270_1446 (size=12697) 2024-12-08T03:42:08,625 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/20add23d751843a7844d3ca8a53299cc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/20add23d751843a7844d3ca8a53299cc 2024-12-08T03:42:08,628 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into 20add23d751843a7844d3ca8a53299cc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:08,628 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:08,628 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629328598; duration=0sec 2024-12-08T03:42:08,629 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:08,629 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:08,629 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:08,630 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:08,630 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:08,630 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:08,631 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ed2faae1c7484ba69bf465870b7e97c8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/040a538e465845d4858d1d5fd723e2ed, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.0 K 2024-12-08T03:42:08,631 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed2faae1c7484ba69bf465870b7e97c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733629324394 2024-12-08T03:42:08,631 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040a538e465845d4858d1d5fd723e2ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733629325199 2024-12-08T03:42:08,631 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae6e7cdcb744e4ab3ae12a5d8bc422d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733629326602 2024-12-08T03:42:08,637 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:08,637 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/d77d27b2a0524d3eb61af35c632803b2 is 50, key is test_row_0/C:col10/1733629327751/Put/seqid=0 2024-12-08T03:42:08,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742271_1447 (size=12697) 2024-12-08T03:42:08,644 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/d77d27b2a0524d3eb61af35c632803b2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d77d27b2a0524d3eb61af35c632803b2 2024-12-08T03:42:08,648 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into d77d27b2a0524d3eb61af35c632803b2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:08,648 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:08,648 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629328598; duration=0sec 2024-12-08T03:42:08,648 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:08,648 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T03:42:08,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T03:42:08,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:08,754 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:42:08,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/955a23631a044c93b2f8daf640ab14e4 is 50, key is test_row_0/A:col10/1733629327760/Put/seqid=0 2024-12-08T03:42:08,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742272_1448 (size=12151) 2024-12-08T03:42:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T03:42:08,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629388939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629388945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629388946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629388947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,025 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/2c727b080a7645c681d99594eb6c6150 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/2c727b080a7645c681d99594eb6c6150 2024-12-08T03:42:09,029 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into 2c727b080a7645c681d99594eb6c6150(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:09,029 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:09,029 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629328598; duration=0sec 2024-12-08T03:42:09,029 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:09,029 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:09,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629389058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629389058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629389058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629389068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,161 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/955a23631a044c93b2f8daf640ab14e4 2024-12-08T03:42:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a7447de467b34a438785417f5adb978c is 50, key is test_row_0/B:col10/1733629327760/Put/seqid=0 2024-12-08T03:42:09,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742273_1449 (size=12151) 2024-12-08T03:42:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T03:42:09,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629389278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629389278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629389279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629389279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629389428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,430 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,570 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a7447de467b34a438785417f5adb978c 2024-12-08T03:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/ef176d3c451749fa8b8dc89ee29f09cf is 50, key is test_row_0/C:col10/1733629327760/Put/seqid=0 2024-12-08T03:42:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742274_1450 (size=12151) 2024-12-08T03:42:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629389588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629389588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629389599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629389599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T03:42:09,980 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/ef176d3c451749fa8b8dc89ee29f09cf 2024-12-08T03:42:09,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/955a23631a044c93b2f8daf640ab14e4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/955a23631a044c93b2f8daf640ab14e4 2024-12-08T03:42:09,987 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/955a23631a044c93b2f8daf640ab14e4, entries=150, sequenceid=247, filesize=11.9 K 2024-12-08T03:42:09,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a7447de467b34a438785417f5adb978c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a7447de467b34a438785417f5adb978c 2024-12-08T03:42:09,990 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a7447de467b34a438785417f5adb978c, entries=150, sequenceid=247, filesize=11.9 K 2024-12-08T03:42:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/ef176d3c451749fa8b8dc89ee29f09cf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ef176d3c451749fa8b8dc89ee29f09cf 2024-12-08T03:42:09,994 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ef176d3c451749fa8b8dc89ee29f09cf, entries=150, sequenceid=247, filesize=11.9 K 2024-12-08T03:42:09,995 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5a84826978f2928496b58ad318fa3531 in 1242ms, sequenceid=247, compaction requested=false 2024-12-08T03:42:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-08T03:42:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-08T03:42:09,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-08T03:42:09,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-12-08T03:42:09,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3990 sec 2024-12-08T03:42:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:10,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:10,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-08T03:42:10,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ddc2870fc5b441f99a99e62f284a2955 is 50, key is test_row_0/A:col10/1733629330101/Put/seqid=0 2024-12-08T03:42:10,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742275_1451 (size=12301) 2024-12-08T03:42:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629390113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629390115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629390119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629390120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629390228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629390228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629390229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629390238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629390438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629390438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629390438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629390448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ddc2870fc5b441f99a99e62f284a2955 2024-12-08T03:42:10,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/5a7c1bb0ea3446f588680bbe9687ad65 is 50, key is test_row_0/B:col10/1733629330101/Put/seqid=0 2024-12-08T03:42:10,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742276_1452 (size=12301) 2024-12-08T03:42:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T03:42:10,748 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:10,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-08T03:42:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T03:42:10,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:10,751 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:10,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:10,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629390748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629390748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629390748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629390759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T03:42:10,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T03:42:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:10,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:10,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/5a7c1bb0ea3446f588680bbe9687ad65 2024-12-08T03:42:10,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8c7d4c12b5414186917f28fffdc9aecc is 50, key is test_row_0/C:col10/1733629330101/Put/seqid=0 2024-12-08T03:42:10,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742277_1453 (size=12301) 2024-12-08T03:42:10,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8c7d4c12b5414186917f28fffdc9aecc 2024-12-08T03:42:10,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/ddc2870fc5b441f99a99e62f284a2955 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ddc2870fc5b441f99a99e62f284a2955 2024-12-08T03:42:10,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ddc2870fc5b441f99a99e62f284a2955, entries=150, sequenceid=275, filesize=12.0 K 2024-12-08T03:42:10,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/5a7c1bb0ea3446f588680bbe9687ad65 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a7c1bb0ea3446f588680bbe9687ad65 2024-12-08T03:42:10,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a7c1bb0ea3446f588680bbe9687ad65, entries=150, sequenceid=275, filesize=12.0 K 2024-12-08T03:42:10,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/8c7d4c12b5414186917f28fffdc9aecc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8c7d4c12b5414186917f28fffdc9aecc 2024-12-08T03:42:10,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8c7d4c12b5414186917f28fffdc9aecc, entries=150, sequenceid=275, filesize=12.0 K 2024-12-08T03:42:10,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5a84826978f2928496b58ad318fa3531 in 843ms, sequenceid=275, compaction requested=true 2024-12-08T03:42:10,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:10,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:10,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:10,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:10,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:10,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:10,946 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:10,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:10,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:10,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:10,946 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:10,946 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:10,946 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:10,946 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:10,946 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:10,947 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/2c727b080a7645c681d99594eb6c6150, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a7447de467b34a438785417f5adb978c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a7c1bb0ea3446f588680bbe9687ad65] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.3 K 2024-12-08T03:42:10,947 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/20add23d751843a7844d3ca8a53299cc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/955a23631a044c93b2f8daf640ab14e4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ddc2870fc5b441f99a99e62f284a2955] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.3 K 2024-12-08T03:42:10,947 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20add23d751843a7844d3ca8a53299cc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733629326602 2024-12-08T03:42:10,947 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c727b080a7645c681d99594eb6c6150, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733629326602 2024-12-08T03:42:10,947 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955a23631a044c93b2f8daf640ab14e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733629327760 2024-12-08T03:42:10,947 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a7447de467b34a438785417f5adb978c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733629327760 2024-12-08T03:42:10,947 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc2870fc5b441f99a99e62f284a2955, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733629328944 2024-12-08T03:42:10,947 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a7c1bb0ea3446f588680bbe9687ad65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733629328944 2024-12-08T03:42:10,966 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:10,966 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/bfd197bb80ce4f3185f359a1d1b0251c is 50, key is test_row_0/A:col10/1733629330101/Put/seqid=0 2024-12-08T03:42:10,968 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:10,968 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/662239feb5704deb80fef3413f504c02 is 50, key is test_row_0/B:col10/1733629330101/Put/seqid=0 2024-12-08T03:42:10,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742279_1455 (size=12949) 2024-12-08T03:42:10,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742278_1454 (size=12949) 2024-12-08T03:42:11,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T03:42:11,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:11,056 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-08T03:42:11,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/cd5762cf971a48f78b6271dd10667e97 is 50, key is test_row_0/A:col10/1733629330118/Put/seqid=0 2024-12-08T03:42:11,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742280_1456 (size=12301) 2024-12-08T03:42:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T03:42:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:11,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:11,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629391299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629391300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629391301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629391302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T03:42:11,383 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/662239feb5704deb80fef3413f504c02 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/662239feb5704deb80fef3413f504c02 2024-12-08T03:42:11,383 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/bfd197bb80ce4f3185f359a1d1b0251c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bfd197bb80ce4f3185f359a1d1b0251c 2024-12-08T03:42:11,387 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into 662239feb5704deb80fef3413f504c02(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:11,387 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:11,387 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629330946; duration=0sec 2024-12-08T03:42:11,387 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:11,388 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:11,388 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:11,388 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into bfd197bb80ce4f3185f359a1d1b0251c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:11,388 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:11,388 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629330945; duration=0sec 2024-12-08T03:42:11,388 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:11,388 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:11,389 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:11,389 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:11,389 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:11,389 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d77d27b2a0524d3eb61af35c632803b2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ef176d3c451749fa8b8dc89ee29f09cf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8c7d4c12b5414186917f28fffdc9aecc] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.3 K 2024-12-08T03:42:11,389 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d77d27b2a0524d3eb61af35c632803b2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733629326602 2024-12-08T03:42:11,390 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ef176d3c451749fa8b8dc89ee29f09cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733629327760 2024-12-08T03:42:11,390 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7d4c12b5414186917f28fffdc9aecc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733629328944 2024-12-08T03:42:11,394 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#390 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:11,395 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/0e225d8a3bd64846a1d3ec47786c7f39 is 50, key is test_row_0/C:col10/1733629330101/Put/seqid=0 2024-12-08T03:42:11,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742281_1457 (size=12949) 2024-12-08T03:42:11,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629391408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629391418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629391418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629391419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,467 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/cd5762cf971a48f78b6271dd10667e97 2024-12-08T03:42:11,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/fb734e198d1344bf9eedd2c6d86b3754 is 50, key is test_row_0/B:col10/1733629330118/Put/seqid=0 2024-12-08T03:42:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742282_1458 (size=12301) 2024-12-08T03:42:11,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629391618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629391628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629391628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629391628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,801 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/0e225d8a3bd64846a1d3ec47786c7f39 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0e225d8a3bd64846a1d3ec47786c7f39 2024-12-08T03:42:11,805 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into 0e225d8a3bd64846a1d3ec47786c7f39(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:11,805 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:11,805 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629330946; duration=0sec 2024-12-08T03:42:11,805 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:11,805 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:11,875 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/fb734e198d1344bf9eedd2c6d86b3754 2024-12-08T03:42:11,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/9de743e56512498e935c1a38cddd0715 is 50, key is test_row_0/C:col10/1733629330118/Put/seqid=0 2024-12-08T03:42:11,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742283_1459 (size=12301) 2024-12-08T03:42:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T03:42:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629391928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629391938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629391938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629391938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,284 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/9de743e56512498e935c1a38cddd0715 2024-12-08T03:42:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/cd5762cf971a48f78b6271dd10667e97 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/cd5762cf971a48f78b6271dd10667e97 2024-12-08T03:42:12,290 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/cd5762cf971a48f78b6271dd10667e97, entries=150, sequenceid=284, filesize=12.0 K 2024-12-08T03:42:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/fb734e198d1344bf9eedd2c6d86b3754 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/fb734e198d1344bf9eedd2c6d86b3754 2024-12-08T03:42:12,293 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/fb734e198d1344bf9eedd2c6d86b3754, entries=150, sequenceid=284, filesize=12.0 K 2024-12-08T03:42:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/9de743e56512498e935c1a38cddd0715 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9de743e56512498e935c1a38cddd0715 2024-12-08T03:42:12,297 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9de743e56512498e935c1a38cddd0715, entries=150, sequenceid=284, filesize=12.0 K 2024-12-08T03:42:12,297 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 5a84826978f2928496b58ad318fa3531 in 1241ms, sequenceid=284, compaction requested=false 2024-12-08T03:42:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-08T03:42:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-08T03:42:12,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-08T03:42:12,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5480 sec 2024-12-08T03:42:12,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5520 sec 2024-12-08T03:42:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:12,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:12,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-08T03:42:12,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/76c0b314ec374847a3f6d4bafbaaea9d is 50, key is test_row_0/A:col10/1733629331300/Put/seqid=0 2024-12-08T03:42:12,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742284_1460 (size=14741) 2024-12-08T03:42:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629392449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629392453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629392454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629392454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629392558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629392558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629392558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629392568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629392768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629392768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629392768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629392778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:12,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/76c0b314ec374847a3f6d4bafbaaea9d 2024-12-08T03:42:12,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/810472fc027348e889230d63861170db is 50, key is test_row_0/B:col10/1733629331300/Put/seqid=0 2024-12-08T03:42:12,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742285_1461 (size=12301) 2024-12-08T03:42:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T03:42:12,898 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:12,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-08T03:42:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T03:42:12,900 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:12,900 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:12,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T03:42:13,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T03:42:13,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:13,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629393078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629393078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629393078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629393088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T03:42:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:13,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T03:42:13,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/810472fc027348e889230d63861170db 2024-12-08T03:42:13,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/1cce547387ac420dbd215c547379412c is 50, key is test_row_0/C:col10/1733629331300/Put/seqid=0 2024-12-08T03:42:13,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742286_1462 (size=12301) 2024-12-08T03:42:13,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T03:42:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59688 deadline: 1733629393458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,463 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in 5a84826978f2928496b58ad318fa3531 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T03:42:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T03:42:13,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629393598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629393598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:13,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T03:42:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:13,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/1cce547387ac420dbd215c547379412c 2024-12-08T03:42:13,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/76c0b314ec374847a3f6d4bafbaaea9d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/76c0b314ec374847a3f6d4bafbaaea9d 2024-12-08T03:42:13,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/76c0b314ec374847a3f6d4bafbaaea9d, entries=200, sequenceid=315, filesize=14.4 K 2024-12-08T03:42:13,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/810472fc027348e889230d63861170db as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/810472fc027348e889230d63861170db 2024-12-08T03:42:13,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/810472fc027348e889230d63861170db, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T03:42:13,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/1cce547387ac420dbd215c547379412c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1cce547387ac420dbd215c547379412c 2024-12-08T03:42:13,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1cce547387ac420dbd215c547379412c, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T03:42:13,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 5a84826978f2928496b58ad318fa3531 in 1248ms, sequenceid=315, compaction requested=true 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:13,690 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:13,690 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:13,691 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:13,691 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:13,691 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,691 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bfd197bb80ce4f3185f359a1d1b0251c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/cd5762cf971a48f78b6271dd10667e97, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/76c0b314ec374847a3f6d4bafbaaea9d] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=39.1 K 2024-12-08T03:42:13,692 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd197bb80ce4f3185f359a1d1b0251c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733629328944 2024-12-08T03:42:13,693 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:13,693 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:13,693 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5762cf971a48f78b6271dd10667e97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733629330109 2024-12-08T03:42:13,693 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,693 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/662239feb5704deb80fef3413f504c02, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/fb734e198d1344bf9eedd2c6d86b3754, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/810472fc027348e889230d63861170db] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.7 K 2024-12-08T03:42:13,695 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 662239feb5704deb80fef3413f504c02, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733629328944 2024-12-08T03:42:13,695 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c0b314ec374847a3f6d4bafbaaea9d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733629331278 2024-12-08T03:42:13,695 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting fb734e198d1344bf9eedd2c6d86b3754, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733629330109 2024-12-08T03:42:13,695 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 810472fc027348e889230d63861170db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733629331299 2024-12-08T03:42:13,701 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:13,701 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/21e76afbb8d849d399b7bc27599e680e is 50, key is test_row_0/A:col10/1733629331300/Put/seqid=0 2024-12-08T03:42:13,712 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:13,713 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/eb2a87e4b8f14b61a7efba2cf991ea45 is 50, key is test_row_0/B:col10/1733629331300/Put/seqid=0 2024-12-08T03:42:13,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742287_1463 (size=13051) 2024-12-08T03:42:13,720 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/21e76afbb8d849d399b7bc27599e680e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/21e76afbb8d849d399b7bc27599e680e 2024-12-08T03:42:13,724 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into 21e76afbb8d849d399b7bc27599e680e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:13,724 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:13,724 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629333690; duration=0sec 2024-12-08T03:42:13,724 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:13,724 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:13,724 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:13,726 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:13,726 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:13,726 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,726 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0e225d8a3bd64846a1d3ec47786c7f39, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9de743e56512498e935c1a38cddd0715, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1cce547387ac420dbd215c547379412c] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.7 K 2024-12-08T03:42:13,726 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e225d8a3bd64846a1d3ec47786c7f39, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733629328944 2024-12-08T03:42:13,728 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de743e56512498e935c1a38cddd0715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733629330109 2024-12-08T03:42:13,728 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cce547387ac420dbd215c547379412c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733629331299 2024-12-08T03:42:13,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742288_1464 (size=13051) 2024-12-08T03:42:13,744 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:13,745 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/e104ad5370214a2786446126042ed699 is 50, key is test_row_0/C:col10/1733629331300/Put/seqid=0 2024-12-08T03:42:13,750 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/eb2a87e4b8f14b61a7efba2cf991ea45 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/eb2a87e4b8f14b61a7efba2cf991ea45 2024-12-08T03:42:13,755 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into eb2a87e4b8f14b61a7efba2cf991ea45(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:13,755 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:13,755 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629333690; duration=0sec 2024-12-08T03:42:13,755 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:13,755 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742289_1465 (size=13051) 2024-12-08T03:42:13,762 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/e104ad5370214a2786446126042ed699 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e104ad5370214a2786446126042ed699 2024-12-08T03:42:13,765 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into e104ad5370214a2786446126042ed699(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:13,765 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:13,766 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629333690; duration=0sec 2024-12-08T03:42:13,766 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:13,766 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:13,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T03:42:13,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:13,815 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-08T03:42:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/92a22a3218e3409780ec4b6b3deb8e62 is 50, key is test_row_0/A:col10/1733629332448/Put/seqid=0 2024-12-08T03:42:13,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742290_1466 (size=12301) 2024-12-08T03:42:13,822 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/92a22a3218e3409780ec4b6b3deb8e62 2024-12-08T03:42:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/22479235710140408602e13080c3a1d0 is 50, key is test_row_0/B:col10/1733629332448/Put/seqid=0 2024-12-08T03:42:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742291_1467 (size=12301) 2024-12-08T03:42:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T03:42:14,231 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/22479235710140408602e13080c3a1d0 2024-12-08T03:42:14,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/f3e41d8bb8884e7cb0ba947ee4955f98 is 50, key is test_row_0/C:col10/1733629332448/Put/seqid=0 2024-12-08T03:42:14,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742292_1468 (size=12301) 2024-12-08T03:42:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:14,640 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/f3e41d8bb8884e7cb0ba947ee4955f98 2024-12-08T03:42:14,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/92a22a3218e3409780ec4b6b3deb8e62 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/92a22a3218e3409780ec4b6b3deb8e62 2024-12-08T03:42:14,646 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/92a22a3218e3409780ec4b6b3deb8e62, entries=150, sequenceid=326, filesize=12.0 K 2024-12-08T03:42:14,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/22479235710140408602e13080c3a1d0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/22479235710140408602e13080c3a1d0 2024-12-08T03:42:14,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629394640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,651 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/22479235710140408602e13080c3a1d0, entries=150, sequenceid=326, filesize=12.0 K 2024-12-08T03:42:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/f3e41d8bb8884e7cb0ba947ee4955f98 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f3e41d8bb8884e7cb0ba947ee4955f98 2024-12-08T03:42:14,654 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f3e41d8bb8884e7cb0ba947ee4955f98, entries=150, sequenceid=326, filesize=12.0 K 2024-12-08T03:42:14,655 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 5a84826978f2928496b58ad318fa3531 in 840ms, sequenceid=326, compaction requested=false 2024-12-08T03:42:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-08T03:42:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-08T03:42:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:14,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:14,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-12-08T03:42:14,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-08T03:42:14,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7560 sec 2024-12-08T03:42:14,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7600 sec 2024-12-08T03:42:14,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/f9b66ec9b8524a689d8a6c02b0f12463 is 50, key is test_row_0/A:col10/1733629334656/Put/seqid=0 2024-12-08T03:42:14,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629394658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629394659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629394660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742293_1469 (size=17181) 2024-12-08T03:42:14,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629394758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629394768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629394768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629394768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629394968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629394978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629394978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629394978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T03:42:15,048 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:15,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-08T03:42:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:15,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:15,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:15,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:15,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/f9b66ec9b8524a689d8a6c02b0f12463 2024-12-08T03:42:15,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/ebda574d5fd242a6a0505f0d2f521726 is 50, key is test_row_0/B:col10/1733629334656/Put/seqid=0 2024-12-08T03:42:15,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742294_1470 (size=12301) 2024-12-08T03:42:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:15,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T03:42:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629395278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629395288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629395288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629395288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T03:42:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:15,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/ebda574d5fd242a6a0505f0d2f521726 2024-12-08T03:42:15,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/91d1f7da6adc4cbfa1540771a2b2e3b6 is 50, key is test_row_0/C:col10/1733629334656/Put/seqid=0 2024-12-08T03:42:15,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742295_1471 (size=12301) 2024-12-08T03:42:15,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T03:42:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T03:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59632 deadline: 1733629395788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733629395798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59618 deadline: 1733629395798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1733629395798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 , the old value is region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., hostname=32df4b4bb439,42629,1733629194165, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:42:15,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T03:42:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. as already flushing 2024-12-08T03:42:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:15,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/91d1f7da6adc4cbfa1540771a2b2e3b6 2024-12-08T03:42:15,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/f9b66ec9b8524a689d8a6c02b0f12463 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f9b66ec9b8524a689d8a6c02b0f12463 2024-12-08T03:42:15,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f9b66ec9b8524a689d8a6c02b0f12463, entries=250, sequenceid=357, filesize=16.8 K 2024-12-08T03:42:15,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/ebda574d5fd242a6a0505f0d2f521726 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/ebda574d5fd242a6a0505f0d2f521726 2024-12-08T03:42:15,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/ebda574d5fd242a6a0505f0d2f521726, entries=150, sequenceid=357, filesize=12.0 K 2024-12-08T03:42:15,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/91d1f7da6adc4cbfa1540771a2b2e3b6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/91d1f7da6adc4cbfa1540771a2b2e3b6 2024-12-08T03:42:15,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/91d1f7da6adc4cbfa1540771a2b2e3b6, entries=150, sequenceid=357, filesize=12.0 K 2024-12-08T03:42:15,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 5a84826978f2928496b58ad318fa3531 in 1244ms, sequenceid=357, compaction requested=true 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:15,902 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a84826978f2928496b58ad318fa3531:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:15,902 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:15,903 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:15,903 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/A is initiating minor compaction (all files) 2024-12-08T03:42:15,903 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:15,903 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/A in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,903 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/B is initiating minor compaction (all files) 2024-12-08T03:42:15,903 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/B in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,903 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/21e76afbb8d849d399b7bc27599e680e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/92a22a3218e3409780ec4b6b3deb8e62, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f9b66ec9b8524a689d8a6c02b0f12463] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=41.5 K 2024-12-08T03:42:15,903 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/eb2a87e4b8f14b61a7efba2cf991ea45, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/22479235710140408602e13080c3a1d0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/ebda574d5fd242a6a0505f0d2f521726] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.8 K 2024-12-08T03:42:15,903 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e76afbb8d849d399b7bc27599e680e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733629331299 2024-12-08T03:42:15,903 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2a87e4b8f14b61a7efba2cf991ea45, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733629331299 2024-12-08T03:42:15,904 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 22479235710140408602e13080c3a1d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733629332448 2024-12-08T03:42:15,904 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a22a3218e3409780ec4b6b3deb8e62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733629332448 2024-12-08T03:42:15,904 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ebda574d5fd242a6a0505f0d2f521726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733629334647 2024-12-08T03:42:15,904 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9b66ec9b8524a689d8a6c02b0f12463, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733629334639 2024-12-08T03:42:15,910 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:15,910 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/d9efaddbdfe64cbfa5b0d40abe3ae73c is 50, key is test_row_0/A:col10/1733629334656/Put/seqid=0 2024-12-08T03:42:15,912 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:15,912 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/e1168a7f4b4640b9a3825ba2b10e53fd is 50, key is test_row_0/B:col10/1733629334656/Put/seqid=0 2024-12-08T03:42:15,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742297_1473 (size=13153) 2024-12-08T03:42:15,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742296_1472 (size=13153) 2024-12-08T03:42:15,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T03:42:15,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:15,964 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-12-08T03:42:15,966 INFO [Thread-1818 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1818. 2024-12-08T03:42:15,966 DEBUG [Thread-1818 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:15,966 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:15,966 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,968 INFO [Thread-1816 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1816. 2024-12-08T03:42:15,968 DEBUG [Thread-1816 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1 is 50, key is test_row_0/A:col10/1733629334659/Put/seqid=0 2024-12-08T03:42:15,968 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,968 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:15,968 INFO [Thread-1814 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1814. 2024-12-08T03:42:15,968 DEBUG [Thread-1814 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:15,968 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,968 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:15,970 INFO [Thread-1810 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1810. 2024-12-08T03:42:15,970 DEBUG [Thread-1810 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:15,970 DEBUG [Thread-1810 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,970 DEBUG [Thread-1810 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:15,970 INFO [Thread-1812 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1812. 2024-12-08T03:42:15,970 DEBUG [Thread-1812 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:15,970 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,970 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:15,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:15,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742298_1474 (size=9857) 2024-12-08T03:42:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:16,333 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/e1168a7f4b4640b9a3825ba2b10e53fd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/e1168a7f4b4640b9a3825ba2b10e53fd 2024-12-08T03:42:16,334 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/d9efaddbdfe64cbfa5b0d40abe3ae73c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d9efaddbdfe64cbfa5b0d40abe3ae73c 2024-12-08T03:42:16,338 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/B of 5a84826978f2928496b58ad318fa3531 into e1168a7f4b4640b9a3825ba2b10e53fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:16,338 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:16,338 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/B, priority=13, startTime=1733629335902; duration=0sec 2024-12-08T03:42:16,339 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:16,339 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:B 2024-12-08T03:42:16,339 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:16,339 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/A of 5a84826978f2928496b58ad318fa3531 into d9efaddbdfe64cbfa5b0d40abe3ae73c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:16,339 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:16,339 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/A, priority=13, startTime=1733629335902; duration=0sec 2024-12-08T03:42:16,339 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:16,339 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:A 2024-12-08T03:42:16,340 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:16,340 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): 5a84826978f2928496b58ad318fa3531/C is initiating minor compaction (all files) 2024-12-08T03:42:16,340 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a84826978f2928496b58ad318fa3531/C in TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:16,340 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e104ad5370214a2786446126042ed699, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f3e41d8bb8884e7cb0ba947ee4955f98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/91d1f7da6adc4cbfa1540771a2b2e3b6] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp, totalSize=36.8 K 2024-12-08T03:42:16,340 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting e104ad5370214a2786446126042ed699, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733629331299 2024-12-08T03:42:16,341 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e41d8bb8884e7cb0ba947ee4955f98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733629332448 2024-12-08T03:42:16,341 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d1f7da6adc4cbfa1540771a2b2e3b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733629334647 2024-12-08T03:42:16,347 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a84826978f2928496b58ad318fa3531#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:16,347 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/f67fd102a33b48238445d2e15c43a44c is 50, key is test_row_0/C:col10/1733629334656/Put/seqid=0 2024-12-08T03:42:16,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742299_1475 (size=13153) 2024-12-08T03:42:16,372 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1 2024-12-08T03:42:16,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a3de085b57964389afa74f3edbee9111 is 50, key is test_row_0/B:col10/1733629334659/Put/seqid=0 2024-12-08T03:42:16,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742300_1476 (size=9857) 2024-12-08T03:42:16,755 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/f67fd102a33b48238445d2e15c43a44c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f67fd102a33b48238445d2e15c43a44c 2024-12-08T03:42:16,759 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a84826978f2928496b58ad318fa3531/C of 5a84826978f2928496b58ad318fa3531 into f67fd102a33b48238445d2e15c43a44c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:16,759 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:16,759 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., storeName=5a84826978f2928496b58ad318fa3531/C, priority=13, startTime=1733629335902; duration=0sec 2024-12-08T03:42:16,759 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:16,759 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a84826978f2928496b58ad318fa3531:C 2024-12-08T03:42:16,787 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a3de085b57964389afa74f3edbee9111 2024-12-08T03:42:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/071468e74b0f497f91151c74b4adfd47 is 50, key is test_row_0/C:col10/1733629334659/Put/seqid=0 2024-12-08T03:42:16,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742301_1477 (size=9857) 2024-12-08T03:42:16,814 INFO [Thread-1799 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1799. 2024-12-08T03:42:16,814 DEBUG [Thread-1799 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:16,814 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,814 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:16,820 INFO [Thread-1805 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1805. 2024-12-08T03:42:16,820 DEBUG [Thread-1805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:16,820 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,820 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:16,820 INFO [Thread-1807 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1807. 2024-12-08T03:42:16,820 INFO [Thread-1801 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1801. 2024-12-08T03:42:16,820 DEBUG [Thread-1807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:16,820 DEBUG [Thread-1801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:16,820 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,820 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,820 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,820 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:16,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:16,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:17,202 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/071468e74b0f497f91151c74b4adfd47 2024-12-08T03:42:17,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1 2024-12-08T03:42:17,219 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1, entries=100, sequenceid=362, filesize=9.6 K 2024-12-08T03:42:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/a3de085b57964389afa74f3edbee9111 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a3de085b57964389afa74f3edbee9111 2024-12-08T03:42:17,223 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a3de085b57964389afa74f3edbee9111, entries=100, sequenceid=362, filesize=9.6 K 2024-12-08T03:42:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/071468e74b0f497f91151c74b4adfd47 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/071468e74b0f497f91151c74b4adfd47 2024-12-08T03:42:17,227 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/071468e74b0f497f91151c74b4adfd47, entries=100, sequenceid=362, filesize=9.6 K 2024-12-08T03:42:17,228 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=26.84 KB/27480 for 5a84826978f2928496b58ad318fa3531 in 1264ms, sequenceid=362, compaction requested=false 2024-12-08T03:42:17,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5a84826978f2928496b58ad318fa3531: 2024-12-08T03:42:17,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:17,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-08T03:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-08T03:42:17,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-08T03:42:17,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1790 sec 2024-12-08T03:42:17,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1820 sec 2024-12-08T03:42:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T03:42:19,219 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:22,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:42:23,573 INFO [Thread-1803 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1803. 2024-12-08T03:42:23,573 DEBUG [Thread-1803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:23,574 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:23,574 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:23,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1957 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5871 rows 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1962 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5886 rows 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1958 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5871 rows 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1968 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5901 rows 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1974 2024-12-08T03:42:23,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5921 rows 2024-12-08T03:42:23,576 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:42:23,576 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:42:23,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:23,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:23,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:23,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T03:42:23,582 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629343582"}]},"ts":"1733629343582"} 2024-12-08T03:42:23,583 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:42:23,583 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:42:23,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:42:23,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, UNASSIGN}] 2024-12-08T03:42:23,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, UNASSIGN 2024-12-08T03:42:23,586 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=5a84826978f2928496b58ad318fa3531, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:23,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, UNASSIGN because future has completed 2024-12-08T03:42:23,587 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:42:23,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:42:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T03:42:23,740 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:23,741 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:42:23,741 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 5a84826978f2928496b58ad318fa3531, disabling compactions & flushes 2024-12-08T03:42:23,741 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:23,741 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:23,741 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. after waiting 0 ms 2024-12-08T03:42:23,741 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:23,741 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 5a84826978f2928496b58ad318fa3531 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-08T03:42:23,749 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/c9df003770f44c8f90fcdbaae5ac3a45 is 50, key is test_row_0/A:col10/1733629336819/Put/seqid=0 2024-12-08T03:42:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742302_1478 (size=12301) 2024-12-08T03:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T03:42:24,154 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/c9df003770f44c8f90fcdbaae5ac3a45 2024-12-08T03:42:24,167 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/c857526224d94183b4478a8cd95dc916 is 50, key is test_row_0/B:col10/1733629336819/Put/seqid=0 2024-12-08T03:42:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742303_1479 (size=12301) 2024-12-08T03:42:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T03:42:24,573 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/c857526224d94183b4478a8cd95dc916 2024-12-08T03:42:24,586 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/3defb41d2df24254ae5ab9464fff2603 is 50, key is test_row_0/C:col10/1733629336819/Put/seqid=0 2024-12-08T03:42:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742304_1480 (size=12301) 2024-12-08T03:42:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T03:42:24,990 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/3defb41d2df24254ae5ab9464fff2603 2024-12-08T03:42:24,994 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/A/c9df003770f44c8f90fcdbaae5ac3a45 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/c9df003770f44c8f90fcdbaae5ac3a45 2024-12-08T03:42:24,999 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/c9df003770f44c8f90fcdbaae5ac3a45, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T03:42:24,999 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/B/c857526224d94183b4478a8cd95dc916 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/c857526224d94183b4478a8cd95dc916 2024-12-08T03:42:25,004 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/c857526224d94183b4478a8cd95dc916, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T03:42:25,004 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/.tmp/C/3defb41d2df24254ae5ab9464fff2603 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/3defb41d2df24254ae5ab9464fff2603 2024-12-08T03:42:25,008 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/3defb41d2df24254ae5ab9464fff2603, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T03:42:25,008 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5a84826978f2928496b58ad318fa3531 in 1267ms, sequenceid=373, compaction requested=true 2024-12-08T03:42:25,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/78f5b69b39cb48759bc075baf0384e34, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/65320e541e994fa3b588652a92f0d513, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/7ff7cb355fe542c7918c7b1a162ef102, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/97227e312cc44ad6a67816a80bd5ea79, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ec314efbaf5b4017850a807a9c794ecb, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/264eb332f32249c8ab4583084051e8d5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/137ceda3bac54bba93a2e2f9dd840a28, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f72ec2f1fdff436594c4e78775274688, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/31de2c68045d4bdaa930006c56636ea4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d0774698219448a9963576790e0b64a2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/5c2a076d645a47569712260dbb7b20c3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bee7d76e9be14dcc887f195e1574e71a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/b4bc8360dd914f03ae128fc290e98188, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ad620fbaf354438eb3b100f90c7f8af8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/173a447346074e45b77534c1a336944a, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/1d234bd5d00b4da29ebdb9bd6fd7c789, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/20add23d751843a7844d3ca8a53299cc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/955a23631a044c93b2f8daf640ab14e4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bfd197bb80ce4f3185f359a1d1b0251c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ddc2870fc5b441f99a99e62f284a2955, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/cd5762cf971a48f78b6271dd10667e97, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/76c0b314ec374847a3f6d4bafbaaea9d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/21e76afbb8d849d399b7bc27599e680e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/92a22a3218e3409780ec4b6b3deb8e62, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f9b66ec9b8524a689d8a6c02b0f12463] to archive 2024-12-08T03:42:25,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:42:25,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/78f5b69b39cb48759bc075baf0384e34 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/78f5b69b39cb48759bc075baf0384e34 2024-12-08T03:42:25,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/65320e541e994fa3b588652a92f0d513 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/65320e541e994fa3b588652a92f0d513 2024-12-08T03:42:25,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/7ff7cb355fe542c7918c7b1a162ef102 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/7ff7cb355fe542c7918c7b1a162ef102 2024-12-08T03:42:25,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/97227e312cc44ad6a67816a80bd5ea79 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/97227e312cc44ad6a67816a80bd5ea79 2024-12-08T03:42:25,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ec314efbaf5b4017850a807a9c794ecb to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ec314efbaf5b4017850a807a9c794ecb 2024-12-08T03:42:25,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/264eb332f32249c8ab4583084051e8d5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/264eb332f32249c8ab4583084051e8d5 2024-12-08T03:42:25,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/137ceda3bac54bba93a2e2f9dd840a28 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/137ceda3bac54bba93a2e2f9dd840a28 2024-12-08T03:42:25,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f72ec2f1fdff436594c4e78775274688 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f72ec2f1fdff436594c4e78775274688 2024-12-08T03:42:25,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/31de2c68045d4bdaa930006c56636ea4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/31de2c68045d4bdaa930006c56636ea4 2024-12-08T03:42:25,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d0774698219448a9963576790e0b64a2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d0774698219448a9963576790e0b64a2 2024-12-08T03:42:25,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/5c2a076d645a47569712260dbb7b20c3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/5c2a076d645a47569712260dbb7b20c3 2024-12-08T03:42:25,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bee7d76e9be14dcc887f195e1574e71a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bee7d76e9be14dcc887f195e1574e71a 2024-12-08T03:42:25,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/b4bc8360dd914f03ae128fc290e98188 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/b4bc8360dd914f03ae128fc290e98188 2024-12-08T03:42:25,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ad620fbaf354438eb3b100f90c7f8af8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ad620fbaf354438eb3b100f90c7f8af8 2024-12-08T03:42:25,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/173a447346074e45b77534c1a336944a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/173a447346074e45b77534c1a336944a 2024-12-08T03:42:25,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/1d234bd5d00b4da29ebdb9bd6fd7c789 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/1d234bd5d00b4da29ebdb9bd6fd7c789 2024-12-08T03:42:25,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/20add23d751843a7844d3ca8a53299cc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/20add23d751843a7844d3ca8a53299cc 2024-12-08T03:42:25,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/955a23631a044c93b2f8daf640ab14e4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/955a23631a044c93b2f8daf640ab14e4 2024-12-08T03:42:25,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bfd197bb80ce4f3185f359a1d1b0251c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/bfd197bb80ce4f3185f359a1d1b0251c 2024-12-08T03:42:25,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ddc2870fc5b441f99a99e62f284a2955 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/ddc2870fc5b441f99a99e62f284a2955 2024-12-08T03:42:25,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/cd5762cf971a48f78b6271dd10667e97 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/cd5762cf971a48f78b6271dd10667e97 2024-12-08T03:42:25,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/76c0b314ec374847a3f6d4bafbaaea9d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/76c0b314ec374847a3f6d4bafbaaea9d 2024-12-08T03:42:25,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/21e76afbb8d849d399b7bc27599e680e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/21e76afbb8d849d399b7bc27599e680e 2024-12-08T03:42:25,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/92a22a3218e3409780ec4b6b3deb8e62 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/92a22a3218e3409780ec4b6b3deb8e62 2024-12-08T03:42:25,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f9b66ec9b8524a689d8a6c02b0f12463 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/f9b66ec9b8524a689d8a6c02b0f12463 2024-12-08T03:42:25,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/bd1adec3771d4473bb859463e1587116, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/15cac7c39acc453cb62fbbde6a6548fa, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47b9d694bc7b4ac3a9e507b6e3e2eb09, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a94743bab68457f8352d73babf29b6e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/4973a6dc8af34f7a8409561bfa5fb6ae, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/b1ecdd083c2343dca4a5f06dea6a922f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47f09bc6f42a44e3ac265524d22106bf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1959c344238e4ee1be0cadc213359bb5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1bca9da79a0b4b9a857504b7bcac6ad9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1684fa2f63fc4fb89f48aafb3d4bc17e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/26a081a94fd04cc6a42071639af47bc9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a9ea692c016f4415ace6ec7edb8425cf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/387a7fa1739f40079fa44c87c56428d1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/358eb653f9114db3a6a2ce85fdc6b185, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1d43245e31b54b11a151a22c41b401e2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/2c727b080a7645c681d99594eb6c6150, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/642c77c6a21c47aa8933d043803ff5c9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a7447de467b34a438785417f5adb978c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/662239feb5704deb80fef3413f504c02, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a7c1bb0ea3446f588680bbe9687ad65, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/fb734e198d1344bf9eedd2c6d86b3754, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/eb2a87e4b8f14b61a7efba2cf991ea45, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/810472fc027348e889230d63861170db, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/22479235710140408602e13080c3a1d0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/ebda574d5fd242a6a0505f0d2f521726] to archive 2024-12-08T03:42:25,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:42:25,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/bd1adec3771d4473bb859463e1587116 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/bd1adec3771d4473bb859463e1587116 2024-12-08T03:42:25,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/15cac7c39acc453cb62fbbde6a6548fa to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/15cac7c39acc453cb62fbbde6a6548fa 2024-12-08T03:42:25,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47b9d694bc7b4ac3a9e507b6e3e2eb09 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47b9d694bc7b4ac3a9e507b6e3e2eb09 2024-12-08T03:42:25,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a94743bab68457f8352d73babf29b6e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a94743bab68457f8352d73babf29b6e 2024-12-08T03:42:25,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/4973a6dc8af34f7a8409561bfa5fb6ae to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/4973a6dc8af34f7a8409561bfa5fb6ae 2024-12-08T03:42:25,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/b1ecdd083c2343dca4a5f06dea6a922f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/b1ecdd083c2343dca4a5f06dea6a922f 2024-12-08T03:42:25,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47f09bc6f42a44e3ac265524d22106bf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/47f09bc6f42a44e3ac265524d22106bf 2024-12-08T03:42:25,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1959c344238e4ee1be0cadc213359bb5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1959c344238e4ee1be0cadc213359bb5 2024-12-08T03:42:25,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1bca9da79a0b4b9a857504b7bcac6ad9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1bca9da79a0b4b9a857504b7bcac6ad9 2024-12-08T03:42:25,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1684fa2f63fc4fb89f48aafb3d4bc17e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1684fa2f63fc4fb89f48aafb3d4bc17e 2024-12-08T03:42:25,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/26a081a94fd04cc6a42071639af47bc9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/26a081a94fd04cc6a42071639af47bc9 2024-12-08T03:42:25,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a9ea692c016f4415ace6ec7edb8425cf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a9ea692c016f4415ace6ec7edb8425cf 2024-12-08T03:42:25,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/387a7fa1739f40079fa44c87c56428d1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/387a7fa1739f40079fa44c87c56428d1 2024-12-08T03:42:25,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/358eb653f9114db3a6a2ce85fdc6b185 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/358eb653f9114db3a6a2ce85fdc6b185 2024-12-08T03:42:25,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1d43245e31b54b11a151a22c41b401e2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/1d43245e31b54b11a151a22c41b401e2 2024-12-08T03:42:25,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/2c727b080a7645c681d99594eb6c6150 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/2c727b080a7645c681d99594eb6c6150 2024-12-08T03:42:25,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/642c77c6a21c47aa8933d043803ff5c9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/642c77c6a21c47aa8933d043803ff5c9 2024-12-08T03:42:25,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a7447de467b34a438785417f5adb978c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a7447de467b34a438785417f5adb978c 2024-12-08T03:42:25,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/662239feb5704deb80fef3413f504c02 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/662239feb5704deb80fef3413f504c02 2024-12-08T03:42:25,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a7c1bb0ea3446f588680bbe9687ad65 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/5a7c1bb0ea3446f588680bbe9687ad65 2024-12-08T03:42:25,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/fb734e198d1344bf9eedd2c6d86b3754 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/fb734e198d1344bf9eedd2c6d86b3754 2024-12-08T03:42:25,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/eb2a87e4b8f14b61a7efba2cf991ea45 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/eb2a87e4b8f14b61a7efba2cf991ea45 2024-12-08T03:42:25,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/810472fc027348e889230d63861170db to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/810472fc027348e889230d63861170db 2024-12-08T03:42:25,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/22479235710140408602e13080c3a1d0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/22479235710140408602e13080c3a1d0 2024-12-08T03:42:25,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/ebda574d5fd242a6a0505f0d2f521726 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/ebda574d5fd242a6a0505f0d2f521726 2024-12-08T03:42:25,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e16871089c549309a99d623fab796f1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/2731df018b574ab181749ad5720d23e3, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/7351ce19086244b487f7a23c80ccb000, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/a22833440b6b4886835a828079dacaf0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d2c376176f0a4002b32c0f4ac86420a1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/67d844494da44981a825724c78a8a8cf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0992d90e27d341c5ba90e933940bf978, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/03f396017dc24d4e941018033cf614f5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1721ea9bea274ab0bed9d1b388c9ceec, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e2b204a61a084239b337941b781d2998, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/bbb80ebdb70149e3adb5fe0567f099f0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9b166d3a33c24f70b905b9c777c47fd9, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ed2faae1c7484ba69bf465870b7e97c8, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e3effecc734497081a007c66f5aa1e6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/040a538e465845d4858d1d5fd723e2ed, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d77d27b2a0524d3eb61af35c632803b2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ef176d3c451749fa8b8dc89ee29f09cf, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0e225d8a3bd64846a1d3ec47786c7f39, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8c7d4c12b5414186917f28fffdc9aecc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9de743e56512498e935c1a38cddd0715, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e104ad5370214a2786446126042ed699, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1cce547387ac420dbd215c547379412c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f3e41d8bb8884e7cb0ba947ee4955f98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/91d1f7da6adc4cbfa1540771a2b2e3b6] to archive 2024-12-08T03:42:25,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:42:25,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e16871089c549309a99d623fab796f1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e16871089c549309a99d623fab796f1 2024-12-08T03:42:25,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/2731df018b574ab181749ad5720d23e3 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/2731df018b574ab181749ad5720d23e3 2024-12-08T03:42:25,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/7351ce19086244b487f7a23c80ccb000 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/7351ce19086244b487f7a23c80ccb000 2024-12-08T03:42:25,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/a22833440b6b4886835a828079dacaf0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/a22833440b6b4886835a828079dacaf0 2024-12-08T03:42:25,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d2c376176f0a4002b32c0f4ac86420a1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d2c376176f0a4002b32c0f4ac86420a1 2024-12-08T03:42:25,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/67d844494da44981a825724c78a8a8cf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/67d844494da44981a825724c78a8a8cf 2024-12-08T03:42:25,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0992d90e27d341c5ba90e933940bf978 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0992d90e27d341c5ba90e933940bf978 2024-12-08T03:42:25,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/03f396017dc24d4e941018033cf614f5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/03f396017dc24d4e941018033cf614f5 2024-12-08T03:42:25,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1721ea9bea274ab0bed9d1b388c9ceec to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1721ea9bea274ab0bed9d1b388c9ceec 2024-12-08T03:42:25,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e2b204a61a084239b337941b781d2998 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e2b204a61a084239b337941b781d2998 2024-12-08T03:42:25,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/bbb80ebdb70149e3adb5fe0567f099f0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/bbb80ebdb70149e3adb5fe0567f099f0 2024-12-08T03:42:25,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9b166d3a33c24f70b905b9c777c47fd9 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9b166d3a33c24f70b905b9c777c47fd9 2024-12-08T03:42:25,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ed2faae1c7484ba69bf465870b7e97c8 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ed2faae1c7484ba69bf465870b7e97c8 2024-12-08T03:42:25,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e3effecc734497081a007c66f5aa1e6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8e3effecc734497081a007c66f5aa1e6 2024-12-08T03:42:25,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/040a538e465845d4858d1d5fd723e2ed to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/040a538e465845d4858d1d5fd723e2ed 2024-12-08T03:42:25,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d77d27b2a0524d3eb61af35c632803b2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/d77d27b2a0524d3eb61af35c632803b2 2024-12-08T03:42:25,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/4ae6e7cdcb744e4ab3ae12a5d8bc422d 2024-12-08T03:42:25,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ef176d3c451749fa8b8dc89ee29f09cf to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/ef176d3c451749fa8b8dc89ee29f09cf 2024-12-08T03:42:25,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0e225d8a3bd64846a1d3ec47786c7f39 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/0e225d8a3bd64846a1d3ec47786c7f39 2024-12-08T03:42:25,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8c7d4c12b5414186917f28fffdc9aecc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/8c7d4c12b5414186917f28fffdc9aecc 2024-12-08T03:42:25,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9de743e56512498e935c1a38cddd0715 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/9de743e56512498e935c1a38cddd0715 2024-12-08T03:42:25,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e104ad5370214a2786446126042ed699 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/e104ad5370214a2786446126042ed699 2024-12-08T03:42:25,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1cce547387ac420dbd215c547379412c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/1cce547387ac420dbd215c547379412c 2024-12-08T03:42:25,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f3e41d8bb8884e7cb0ba947ee4955f98 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f3e41d8bb8884e7cb0ba947ee4955f98 2024-12-08T03:42:25,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/91d1f7da6adc4cbfa1540771a2b2e3b6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/91d1f7da6adc4cbfa1540771a2b2e3b6 2024-12-08T03:42:25,078 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-12-08T03:42:25,079 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. 2024-12-08T03:42:25,079 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 5a84826978f2928496b58ad318fa3531: Waiting for close lock at 1733629343741Running coprocessor pre-close hooks at 1733629343741Disabling compacts and flushes for region at 1733629343741Disabling writes for close at 1733629343741Obtaining lock to block concurrent updates at 1733629343742 (+1 ms)Preparing flush snapshotting stores in 5a84826978f2928496b58ad318fa3531 at 1733629343742Finished memstore snapshotting TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733629343742Flushing stores of TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531. at 1733629343743 (+1 ms)Flushing 5a84826978f2928496b58ad318fa3531/A: creating writer at 1733629343743Flushing 5a84826978f2928496b58ad318fa3531/A: appending metadata at 1733629343749 (+6 ms)Flushing 5a84826978f2928496b58ad318fa3531/A: closing flushed file at 1733629343749Flushing 5a84826978f2928496b58ad318fa3531/B: creating writer at 1733629344164 (+415 ms)Flushing 5a84826978f2928496b58ad318fa3531/B: appending metadata at 1733629344166 (+2 ms)Flushing 5a84826978f2928496b58ad318fa3531/B: closing flushed file at 1733629344166Flushing 5a84826978f2928496b58ad318fa3531/C: creating writer at 1733629344583 (+417 ms)Flushing 5a84826978f2928496b58ad318fa3531/C: appending metadata at 1733629344586 (+3 ms)Flushing 5a84826978f2928496b58ad318fa3531/C: closing flushed file at 1733629344586Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bf0df40: reopening flushed file at 1733629344994 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b55bc8f: reopening flushed file at 1733629344999 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@258dcc57: reopening flushed file at 1733629345004 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5a84826978f2928496b58ad318fa3531 in 1267ms, sequenceid=373, compaction requested=true at 1733629345008 (+4 ms)Writing region close event to WAL at 1733629345076 (+68 ms)Running coprocessor post-close hooks at 1733629345079 (+3 ms)Closed at 1733629345079 2024-12-08T03:42:25,081 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:25,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=5a84826978f2928496b58ad318fa3531, regionState=CLOSED 2024-12-08T03:42:25,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:42:25,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-08T03:42:25,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 5a84826978f2928496b58ad318fa3531, server=32df4b4bb439,42629,1733629194165 in 1.4970 sec 2024-12-08T03:42:25,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-08T03:42:25,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a84826978f2928496b58ad318fa3531, UNASSIGN in 1.5030 sec 2024-12-08T03:42:25,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-08T03:42:25,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5060 sec 2024-12-08T03:42:25,091 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629345091"}]},"ts":"1733629345091"} 2024-12-08T03:42:25,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:42:25,092 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:42:25,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5140 sec 2024-12-08T03:42:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T03:42:25,738 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:25,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:42:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,742 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T03:42:25,743 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,746 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:25,749 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/recovered.edits] 2024-12-08T03:42:25,754 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/2b6d1c73f77d4aabb1d9e74a1eedd1e1 2024-12-08T03:42:25,756 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/c9df003770f44c8f90fcdbaae5ac3a45 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/c9df003770f44c8f90fcdbaae5ac3a45 2024-12-08T03:42:25,757 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d9efaddbdfe64cbfa5b0d40abe3ae73c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/A/d9efaddbdfe64cbfa5b0d40abe3ae73c 2024-12-08T03:42:25,761 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a3de085b57964389afa74f3edbee9111 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/a3de085b57964389afa74f3edbee9111 2024-12-08T03:42:25,762 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/c857526224d94183b4478a8cd95dc916 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/c857526224d94183b4478a8cd95dc916 2024-12-08T03:42:25,764 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/e1168a7f4b4640b9a3825ba2b10e53fd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/B/e1168a7f4b4640b9a3825ba2b10e53fd 2024-12-08T03:42:25,767 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/071468e74b0f497f91151c74b4adfd47 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/071468e74b0f497f91151c74b4adfd47 2024-12-08T03:42:25,769 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/3defb41d2df24254ae5ab9464fff2603 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/3defb41d2df24254ae5ab9464fff2603 2024-12-08T03:42:25,771 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f67fd102a33b48238445d2e15c43a44c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/C/f67fd102a33b48238445d2e15c43a44c 2024-12-08T03:42:25,774 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/recovered.edits/376.seqid to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531/recovered.edits/376.seqid 2024-12-08T03:42:25,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/5a84826978f2928496b58ad318fa3531 2024-12-08T03:42:25,775 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:42:25,778 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,780 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:42:25,782 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:42:25,783 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,783 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:42:25,783 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733629345783"}]},"ts":"9223372036854775807"} 2024-12-08T03:42:25,785 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:42:25,785 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5a84826978f2928496b58ad318fa3531, NAME => 'TestAcidGuarantees,,1733629313763.5a84826978f2928496b58ad318fa3531.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:42:25,785 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:42:25,785 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733629345785"}]},"ts":"9223372036854775807"} 2024-12-08T03:42:25,787 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:42:25,787 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-12-08T03:42:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T03:42:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:42:25,849 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:25,862 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 237), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 341), ProcessCount=11 (was 11), AvailableMemoryMB=17078 (was 17151) 2024-12-08T03:42:25,869 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=17078 2024-12-08T03:42:25,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:42:25,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:42:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:25,872 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:42:25,872 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:25,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-08T03:42:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-08T03:42:25,873 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:42:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742305_1481 (size=959) 2024-12-08T03:42:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-08T03:42:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-08T03:42:26,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a 2024-12-08T03:42:26,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742306_1482 (size=53) 2024-12-08T03:42:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-08T03:42:26,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:42:26,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d51381d83515f3cce487395d9d692757, disabling compactions & flushes 2024-12-08T03:42:26,694 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:26,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:26,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. after waiting 0 ms 2024-12-08T03:42:26,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:26,694 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:26,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d51381d83515f3cce487395d9d692757: Waiting for close lock at 1733629346694Disabling compacts and flushes for region at 1733629346694Disabling writes for close at 1733629346694Writing region close event to WAL at 1733629346694Closed at 1733629346694 2024-12-08T03:42:26,697 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:42:26,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733629346697"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733629346697"}]},"ts":"1733629346697"} 2024-12-08T03:42:26,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:42:26,702 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:42:26,702 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629346702"}]},"ts":"1733629346702"} 2024-12-08T03:42:26,704 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:42:26,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, ASSIGN}] 2024-12-08T03:42:26,705 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, ASSIGN 2024-12-08T03:42:26,706 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, ASSIGN; state=OFFLINE, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=false 2024-12-08T03:42:26,857 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:26,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, ASSIGN because future has completed 2024-12-08T03:42:26,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:42:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-08T03:42:27,018 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:27,018 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:42:27,019 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,019 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:42:27,019 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,019 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,020 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,022 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51381d83515f3cce487395d9d692757 columnFamilyName A 2024-12-08T03:42:27,022 DEBUG [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:27,022 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(327): Store=d51381d83515f3cce487395d9d692757/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:42:27,023 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,024 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51381d83515f3cce487395d9d692757 columnFamilyName B 2024-12-08T03:42:27,024 DEBUG [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:27,025 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(327): Store=d51381d83515f3cce487395d9d692757/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:42:27,025 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,026 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51381d83515f3cce487395d9d692757 columnFamilyName C 2024-12-08T03:42:27,026 DEBUG [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:27,026 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(327): Store=d51381d83515f3cce487395d9d692757/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:42:27,027 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,027 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,028 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,029 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,029 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,029 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:42:27,031 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,033 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:42:27,034 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened d51381d83515f3cce487395d9d692757; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62216594, jitterRate=-0.07290050387382507}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:42:27,034 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:27,035 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for d51381d83515f3cce487395d9d692757: Running coprocessor pre-open hook at 1733629347019Writing region info on filesystem at 1733629347019Initializing all the Stores at 1733629347020 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629347020Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629347020Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629347020Cleaning up temporary data from old regions at 1733629347029 (+9 ms)Running coprocessor post-open hooks at 1733629347034 (+5 ms)Region opened successfully at 1733629347035 (+1 ms) 2024-12-08T03:42:27,036 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., pid=140, masterSystemTime=1733629347011 2024-12-08T03:42:27,037 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:27,037 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:27,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:27,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:42:27,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-08T03:42:27,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 in 182 msec 2024-12-08T03:42:27,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-08T03:42:27,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, ASSIGN in 338 msec 2024-12-08T03:42:27,045 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:42:27,045 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629347045"}]},"ts":"1733629347045"} 2024-12-08T03:42:27,046 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:42:27,046 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:42:27,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-12-08T03:42:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-08T03:42:28,029 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:28,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:28,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:28,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:28,035 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517ee2df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:28,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:28,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cd4059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:28,039 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:28,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:28,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:28,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:28,042 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:42:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,46215,1733629193261 2024-12-08T03:42:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10aed64 2024-12-08T03:42:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:42:28,044 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:42:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:42:28,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:42:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:28,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742307_1483 (size=995) 2024-12-08T03:42:28,462 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-08T03:42:28,462 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-08T03:42:28,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:42:28,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, REOPEN/MOVE}] 2024-12-08T03:42:28,473 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, REOPEN/MOVE 2024-12-08T03:42:28,475 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:28,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, REOPEN/MOVE because future has completed 2024-12-08T03:42:28,476 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:42:28,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:42:28,629 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close d51381d83515f3cce487395d9d692757 2024-12-08T03:42:28,630 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:42:28,630 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing d51381d83515f3cce487395d9d692757, disabling compactions & flushes 2024-12-08T03:42:28,630 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:28,630 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:28,630 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. after waiting 0 ms 2024-12-08T03:42:28,630 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:28,638 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T03:42:28,638 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:28,639 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for d51381d83515f3cce487395d9d692757: Waiting for close lock at 1733629348630Running coprocessor pre-close hooks at 1733629348630Disabling compacts and flushes for region at 1733629348630Disabling writes for close at 1733629348630Writing region close event to WAL at 1733629348631 (+1 ms)Running coprocessor post-close hooks at 1733629348638 (+7 ms)Closed at 1733629348638 2024-12-08T03:42:28,639 WARN [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: d51381d83515f3cce487395d9d692757 to self. 2024-12-08T03:42:28,641 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed d51381d83515f3cce487395d9d692757 2024-12-08T03:42:28,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=CLOSED 2024-12-08T03:42:28,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:42:28,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-08T03:42:28,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 in 168 msec 2024-12-08T03:42:28,647 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,42629,1733629194165; forceNewPlan=false, retain=true 2024-12-08T03:42:28,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=OPENING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:28,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, REOPEN/MOVE because future has completed 2024-12-08T03:42:28,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:42:29,011 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,011 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:42:29,012 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,012 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:42:29,012 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,012 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,015 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,017 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51381d83515f3cce487395d9d692757 columnFamilyName A 2024-12-08T03:42:29,019 DEBUG [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:29,019 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(327): Store=d51381d83515f3cce487395d9d692757/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:42:29,020 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,021 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51381d83515f3cce487395d9d692757 columnFamilyName B 2024-12-08T03:42:29,021 DEBUG [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:29,021 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(327): Store=d51381d83515f3cce487395d9d692757/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:42:29,021 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,022 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51381d83515f3cce487395d9d692757 columnFamilyName C 2024-12-08T03:42:29,022 DEBUG [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:29,023 INFO [StoreOpener-d51381d83515f3cce487395d9d692757-1 {}] regionserver.HStore(327): Store=d51381d83515f3cce487395d9d692757/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:42:29,023 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,024 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,025 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,027 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,027 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,027 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:42:29,028 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,029 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened d51381d83515f3cce487395d9d692757; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65298443, jitterRate=-0.026977375149726868}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:42:29,029 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,030 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for d51381d83515f3cce487395d9d692757: Running coprocessor pre-open hook at 1733629349012Writing region info on filesystem at 1733629349012Initializing all the Stores at 1733629349014 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629349014Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629349015 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733629349015Cleaning up temporary data from old regions at 1733629349027 (+12 ms)Running coprocessor post-open hooks at 1733629349029 (+2 ms)Region opened successfully at 1733629349030 (+1 ms) 2024-12-08T03:42:29,031 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., pid=145, masterSystemTime=1733629349004 2024-12-08T03:42:29,032 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,032 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=OPEN, openSeqNum=5, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:42:29,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-08T03:42:29,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 in 232 msec 2024-12-08T03:42:29,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-08T03:42:29,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, REOPEN/MOVE in 566 msec 2024-12-08T03:42:29,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-08T03:42:29,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 573 msec 2024-12-08T03:42:29,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 996 msec 2024-12-08T03:42:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-08T03:42:29,044 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:29,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8e969d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,047 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e586e32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7a7434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,051 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b20accd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,052 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c94eae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d1fd273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b5afcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4d832a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b88cff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3375088a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,060 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3371c31f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193a6a4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ff487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591fe83d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,065 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63087691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,066 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b019e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,067 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d422f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,068 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4273d248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,46215,-1 for getting cluster id 2024-12-08T03:42:29,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:42:29,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '683332fe-025f-4451-984f-68618f51f751' 2024-12-08T03:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "683332fe-025f-4451-984f-68618f51f751" 2024-12-08T03:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f298b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,46215,-1] 2024-12-08T03:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:29,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:42:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d61f5aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:42:29,071 DEBUG [Thread-2180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [Thread-2197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [Thread-2195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [Thread-2184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [Thread-2188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [Thread-2199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,072 DEBUG [Thread-2191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,073 DEBUG [Thread-2193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,073 DEBUG [Thread-2182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,073 DEBUG [Thread-2186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,42629,1733629194165, seqNum=-1] 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:42:29,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:29,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:42:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-08T03:42:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,078 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,080 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5] 2024-12-08T03:42:29,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,084 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:29,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-08T03:42:29,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084afb6ea798dc4bd49985218557fc1234_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629349084/Put/seqid=0 2024-12-08T03:42:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629409102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629409102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629409104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629409106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629409107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742308_1484 (size=12154) 2024-12-08T03:42:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629409208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629409209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629409218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629409218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629409218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T03:42:29,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:29,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T03:42:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:29,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629409418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629409418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629409428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629409428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629409428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:29,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084afb6ea798dc4bd49985218557fc1234_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084afb6ea798dc4bd49985218557fc1234_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:29,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/ac4d046506c14f6fbaf3630365638e17, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:29,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/ac4d046506c14f6fbaf3630365638e17 is 175, key is test_row_0/A:col10/1733629349084/Put/seqid=0 2024-12-08T03:42:29,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742309_1485 (size=30955) 2024-12-08T03:42:29,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T03:42:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T03:42:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:29,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629409728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629409728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629409738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629409738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629409738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:29,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T03:42:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:29,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/ac4d046506c14f6fbaf3630365638e17 2024-12-08T03:42:29,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/b1ea61ef6be84c31a1992f6b7e3bd87d is 50, key is test_row_0/B:col10/1733629349084/Put/seqid=0 2024-12-08T03:42:29,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742310_1486 (size=12001) 2024-12-08T03:42:29,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/b1ea61ef6be84c31a1992f6b7e3bd87d 2024-12-08T03:42:29,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d006959255df4f0aaba2f650e046d59e is 50, key is test_row_0/C:col10/1733629349084/Put/seqid=0 2024-12-08T03:42:29,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742311_1487 (size=12001) 2024-12-08T03:42:29,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d006959255df4f0aaba2f650e046d59e 2024-12-08T03:42:29,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/ac4d046506c14f6fbaf3630365638e17 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17 2024-12-08T03:42:29,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17, entries=150, sequenceid=15, filesize=30.2 K 2024-12-08T03:42:29,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/b1ea61ef6be84c31a1992f6b7e3bd87d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/b1ea61ef6be84c31a1992f6b7e3bd87d 2024-12-08T03:42:29,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/b1ea61ef6be84c31a1992f6b7e3bd87d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:42:29,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d006959255df4f0aaba2f650e046d59e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d006959255df4f0aaba2f650e046d59e 2024-12-08T03:42:29,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d006959255df4f0aaba2f650e046d59e, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:42:29,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for d51381d83515f3cce487395d9d692757 in 895ms, sequenceid=15, compaction requested=false 2024-12-08T03:42:29,979 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:42:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:29,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T03:42:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:29,996 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-08T03:42:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208597393d57d1f4d988bb1b6e7a31fff0f_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629349105/Put/seqid=0 2024-12-08T03:42:30,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742312_1488 (size=12154) 2024-12-08T03:42:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:30,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629410248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629410248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629410248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629410250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629410250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629410358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629410358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:30,407 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208597393d57d1f4d988bb1b6e7a31fff0f_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208597393d57d1f4d988bb1b6e7a31fff0f_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f6c833d6ee3949449304bf12b6001f3c, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f6c833d6ee3949449304bf12b6001f3c is 175, key is test_row_0/A:col10/1733629349105/Put/seqid=0 2024-12-08T03:42:30,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742313_1489 (size=30955) 2024-12-08T03:42:30,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629410569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629410569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,595 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:42:30,812 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f6c833d6ee3949449304bf12b6001f3c 2024-12-08T03:42:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/c724aaacfc9d4ec28169e7a8e23b9508 is 50, key is test_row_0/B:col10/1733629349105/Put/seqid=0 2024-12-08T03:42:30,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742314_1490 (size=12001) 2024-12-08T03:42:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629410878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629410878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:31,220 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/c724aaacfc9d4ec28169e7a8e23b9508 2024-12-08T03:42:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5 is 50, key is test_row_0/C:col10/1733629349105/Put/seqid=0 2024-12-08T03:42:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:31,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742315_1491 (size=12001) 2024-12-08T03:42:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629411258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:31,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629411258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:31,259 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_0' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:31,259 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:31,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629411258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:31,260 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:31,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629411388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629411388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:31,629 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5 2024-12-08T03:42:31,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f6c833d6ee3949449304bf12b6001f3c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c 2024-12-08T03:42:31,636 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c, entries=150, sequenceid=41, filesize=30.2 K 2024-12-08T03:42:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/c724aaacfc9d4ec28169e7a8e23b9508 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/c724aaacfc9d4ec28169e7a8e23b9508 2024-12-08T03:42:31,640 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/c724aaacfc9d4ec28169e7a8e23b9508, entries=150, sequenceid=41, filesize=11.7 K 2024-12-08T03:42:31,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5 2024-12-08T03:42:31,644 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5, entries=150, sequenceid=41, filesize=11.7 K 2024-12-08T03:42:31,644 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for d51381d83515f3cce487395d9d692757 in 1648ms, sequenceid=41, compaction requested=false 2024-12-08T03:42:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-08T03:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-08T03:42:31,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-08T03:42:31,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5660 sec 2024-12-08T03:42:31,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.5730 sec 2024-12-08T03:42:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:32,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:32,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:42:32,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208120ad7657f6e4c02b1b1e473eea1ddd8_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:32,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742316_1492 (size=14594) 2024-12-08T03:42:32,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629412443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:32,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629412443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629412548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629412549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629412758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:32,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629412758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:32,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:32,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208120ad7657f6e4c02b1b1e473eea1ddd8_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208120ad7657f6e4c02b1b1e473eea1ddd8_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:32,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/905bc9e1b87941ee9a991216811c48e0, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:32,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/905bc9e1b87941ee9a991216811c48e0 is 175, key is test_row_0/A:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742317_1493 (size=39549) 2024-12-08T03:42:33,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629413068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629413068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,219 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/905bc9e1b87941ee9a991216811c48e0 2024-12-08T03:42:33,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/3d9c39a84de94e3786a446e14c4a68f6 is 50, key is test_row_0/B:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:33,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742318_1494 (size=12001) 2024-12-08T03:42:33,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/3d9c39a84de94e3786a446e14c4a68f6 2024-12-08T03:42:33,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/515a7cfdff1d4ac2bad012ccdf423696 is 50, key is test_row_0/C:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T03:42:33,239 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:33,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-08T03:42:33,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742319_1495 (size=12001) 2024-12-08T03:42:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T03:42:33,241 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:33,242 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:33,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629413278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629413279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,282 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_0' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,286 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629413288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,290 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T03:42:33,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T03:42:33,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:33,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:33,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:33,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T03:42:33,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T03:42:33,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629413578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629413578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:33,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/515a7cfdff1d4ac2bad012ccdf423696 2024-12-08T03:42:33,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/905bc9e1b87941ee9a991216811c48e0 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0 2024-12-08T03:42:33,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0, entries=200, sequenceid=54, filesize=38.6 K 2024-12-08T03:42:33,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/3d9c39a84de94e3786a446e14c4a68f6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3d9c39a84de94e3786a446e14c4a68f6 2024-12-08T03:42:33,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3d9c39a84de94e3786a446e14c4a68f6, entries=150, sequenceid=54, filesize=11.7 K 2024-12-08T03:42:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/515a7cfdff1d4ac2bad012ccdf423696 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/515a7cfdff1d4ac2bad012ccdf423696 2024-12-08T03:42:33,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/515a7cfdff1d4ac2bad012ccdf423696, entries=150, sequenceid=54, filesize=11.7 K 2024-12-08T03:42:33,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d51381d83515f3cce487395d9d692757 in 1253ms, sequenceid=54, compaction requested=true 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:33,653 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:33,653 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:33,654 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:33,654 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:33,654 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/B is initiating minor compaction (all files) 2024-12-08T03:42:33,654 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/A is initiating minor compaction (all files) 2024-12-08T03:42:33,654 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/B in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,654 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/A in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,654 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/b1ea61ef6be84c31a1992f6b7e3bd87d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/c724aaacfc9d4ec28169e7a8e23b9508, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3d9c39a84de94e3786a446e14c4a68f6] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.2 K 2024-12-08T03:42:33,654 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=99.1 K 2024-12-08T03:42:33,654 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,654 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0] 2024-12-08T03:42:33,654 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ea61ef6be84c31a1992f6b7e3bd87d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629349080 2024-12-08T03:42:33,655 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac4d046506c14f6fbaf3630365638e17, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629349080 2024-12-08T03:42:33,655 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting c724aaacfc9d4ec28169e7a8e23b9508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733629349095 2024-12-08T03:42:33,655 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c833d6ee3949449304bf12b6001f3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733629349095 2024-12-08T03:42:33,655 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9c39a84de94e3786a446e14c4a68f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733629350249 2024-12-08T03:42:33,655 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905bc9e1b87941ee9a991216811c48e0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733629350246 2024-12-08T03:42:33,663 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:33,663 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:33,663 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/230d3d586e204283ab3e9a3b0aa7c834 is 50, key is test_row_0/B:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:33,664 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120801f4482584b640c9a266352ed43398b3_d51381d83515f3cce487395d9d692757 store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:33,665 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120801f4482584b640c9a266352ed43398b3_d51381d83515f3cce487395d9d692757, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:33,666 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120801f4482584b640c9a266352ed43398b3_d51381d83515f3cce487395d9d692757 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:33,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742321_1497 (size=4469) 2024-12-08T03:42:33,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742320_1496 (size=12104) 2024-12-08T03:42:33,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T03:42:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:33,699 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-08T03:42:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bee020c4c3924b2fa31e83701d61e724_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629352431/Put/seqid=0 2024-12-08T03:42:33,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742322_1498 (size=12154) 2024-12-08T03:42:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T03:42:34,070 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#A#compaction#424 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:34,071 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/52a6a6f8c63d4d1593f20f10295816ca is 175, key is test_row_0/A:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:34,073 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/230d3d586e204283ab3e9a3b0aa7c834 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/230d3d586e204283ab3e9a3b0aa7c834 2024-12-08T03:42:34,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742323_1499 (size=31058) 2024-12-08T03:42:34,078 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/B of d51381d83515f3cce487395d9d692757 into 230d3d586e204283ab3e9a3b0aa7c834(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:34,078 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:34,078 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/B, priority=13, startTime=1733629353653; duration=0sec 2024-12-08T03:42:34,078 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:34,078 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:B 2024-12-08T03:42:34,078 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:34,079 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/52a6a6f8c63d4d1593f20f10295816ca as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/52a6a6f8c63d4d1593f20f10295816ca 2024-12-08T03:42:34,079 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:34,079 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/C is initiating minor compaction (all files) 2024-12-08T03:42:34,080 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/C in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:34,080 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d006959255df4f0aaba2f650e046d59e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/515a7cfdff1d4ac2bad012ccdf423696] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.2 K 2024-12-08T03:42:34,080 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d006959255df4f0aaba2f650e046d59e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733629349080 2024-12-08T03:42:34,080 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7208d77dd44b9eb16c1b6bb8a8d2f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733629349095 2024-12-08T03:42:34,080 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 515a7cfdff1d4ac2bad012ccdf423696, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733629350249 2024-12-08T03:42:34,083 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/A of d51381d83515f3cce487395d9d692757 into 52a6a6f8c63d4d1593f20f10295816ca(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:34,083 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:34,083 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/A, priority=13, startTime=1733629353653; duration=0sec 2024-12-08T03:42:34,083 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:34,083 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:A 2024-12-08T03:42:34,087 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:34,087 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/1ab6879254314b3f8683ba5b8ab795ca is 50, key is test_row_0/C:col10/1733629352399/Put/seqid=0 2024-12-08T03:42:34,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742324_1500 (size=12104) 2024-12-08T03:42:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:34,114 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bee020c4c3924b2fa31e83701d61e724_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bee020c4c3924b2fa31e83701d61e724_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:34,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/a7897c2840c843fd9afcd46fef6cb440, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:34,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/a7897c2840c843fd9afcd46fef6cb440 is 175, key is test_row_0/A:col10/1733629352431/Put/seqid=0 2024-12-08T03:42:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742325_1501 (size=30955) 2024-12-08T03:42:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T03:42:34,496 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/1ab6879254314b3f8683ba5b8ab795ca as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1ab6879254314b3f8683ba5b8ab795ca 2024-12-08T03:42:34,500 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/C of d51381d83515f3cce487395d9d692757 into 1ab6879254314b3f8683ba5b8ab795ca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:34,500 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:34,500 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/C, priority=13, startTime=1733629353653; duration=0sec 2024-12-08T03:42:34,500 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:34,500 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:C 2024-12-08T03:42:34,520 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/a7897c2840c843fd9afcd46fef6cb440 2024-12-08T03:42:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/3e81a19e04b14793b09f97a87be36cdd is 50, key is test_row_0/B:col10/1733629352431/Put/seqid=0 2024-12-08T03:42:34,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742326_1502 (size=12001) 2024-12-08T03:42:34,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:34,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629414606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:34,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629414608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629414718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:34,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629414719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:34,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:34,930 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/3e81a19e04b14793b09f97a87be36cdd 2024-12-08T03:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629414928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:34,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629414928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:34,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/947671ba0df34c0ca20b6d4c311b645b is 50, key is test_row_0/C:col10/1733629352431/Put/seqid=0 2024-12-08T03:42:34,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742327_1503 (size=12001) 2024-12-08T03:42:34,939 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/947671ba0df34c0ca20b6d4c311b645b 2024-12-08T03:42:34,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/a7897c2840c843fd9afcd46fef6cb440 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440 2024-12-08T03:42:34,945 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440, entries=150, sequenceid=78, filesize=30.2 K 2024-12-08T03:42:34,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/3e81a19e04b14793b09f97a87be36cdd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3e81a19e04b14793b09f97a87be36cdd 2024-12-08T03:42:34,949 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3e81a19e04b14793b09f97a87be36cdd, entries=150, sequenceid=78, filesize=11.7 K 2024-12-08T03:42:34,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/947671ba0df34c0ca20b6d4c311b645b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/947671ba0df34c0ca20b6d4c311b645b 2024-12-08T03:42:34,954 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/947671ba0df34c0ca20b6d4c311b645b, entries=150, sequenceid=78, filesize=11.7 K 2024-12-08T03:42:34,955 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d51381d83515f3cce487395d9d692757 in 1256ms, sequenceid=78, compaction requested=false 2024-12-08T03:42:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-08T03:42:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-08T03:42:34,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-08T03:42:34,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-12-08T03:42:34,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.7190 sec 2024-12-08T03:42:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:35,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-08T03:42:35,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ed709810ac50409f88f6526499851f59_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:35,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742328_1504 (size=17034) 2024-12-08T03:42:35,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629415288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629415290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T03:42:35,388 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:35,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-08T03:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:35,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:35,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:35,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:35,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629415398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629415398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:35,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:35,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:35,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:35,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:35,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629415608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629415608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,650 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:35,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ed709810ac50409f88f6526499851f59_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ed709810ac50409f88f6526499851f59_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:35,654 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/447212f64442448bb07b53d02413bb71, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:35,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/447212f64442448bb07b53d02413bb71 is 175, key is test_row_0/A:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:35,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742329_1505 (size=48139) 2024-12-08T03:42:35,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:35,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:35,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:35,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:35,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629415918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629415918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:36,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:36,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/447212f64442448bb07b53d02413bb71 2024-12-08T03:42:36,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/1843979fd1014fc18fe1058a503462e2 is 50, key is test_row_0/B:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:36,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742330_1506 (size=12001) 2024-12-08T03:42:36,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629416428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629416428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:36,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/1843979fd1014fc18fe1058a503462e2 2024-12-08T03:42:36,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/254ac1523c0744aa97e25827982fbcc6 is 50, key is test_row_0/C:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742331_1507 (size=12001) 2024-12-08T03:42:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:36,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:36,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:36,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/254ac1523c0744aa97e25827982fbcc6 2024-12-08T03:42:36,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/447212f64442448bb07b53d02413bb71 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71 2024-12-08T03:42:36,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71, entries=250, sequenceid=95, filesize=47.0 K 2024-12-08T03:42:36,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/1843979fd1014fc18fe1058a503462e2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/1843979fd1014fc18fe1058a503462e2 2024-12-08T03:42:36,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/1843979fd1014fc18fe1058a503462e2, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T03:42:36,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/254ac1523c0744aa97e25827982fbcc6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/254ac1523c0744aa97e25827982fbcc6 2024-12-08T03:42:36,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/254ac1523c0744aa97e25827982fbcc6, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T03:42:36,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d51381d83515f3cce487395d9d692757 in 1647ms, sequenceid=95, compaction requested=true 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:36,888 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:36,888 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:36,888 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:36,888 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/A is initiating minor compaction (all files) 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/B is initiating minor compaction (all files) 2024-12-08T03:42:36,889 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/A in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,889 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/B in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,889 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/230d3d586e204283ab3e9a3b0aa7c834, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3e81a19e04b14793b09f97a87be36cdd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/1843979fd1014fc18fe1058a503462e2] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.3 K 2024-12-08T03:42:36,889 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/52a6a6f8c63d4d1593f20f10295816ca, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=107.6 K 2024-12-08T03:42:36,889 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/52a6a6f8c63d4d1593f20f10295816ca, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71] 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a6a6f8c63d4d1593f20f10295816ca, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733629350249 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 230d3d586e204283ab3e9a3b0aa7c834, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733629350249 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7897c2840c843fd9afcd46fef6cb440, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733629352431 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e81a19e04b14793b09f97a87be36cdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733629352431 2024-12-08T03:42:36,889 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1843979fd1014fc18fe1058a503462e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733629354602 2024-12-08T03:42:36,890 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447212f64442448bb07b53d02413bb71, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733629354602 2024-12-08T03:42:36,893 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:36,894 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:36,895 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/41e12b3898b2443b90fb8f54ba2d1cc1 is 50, key is test_row_0/B:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:36,895 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208ca945729dbbe407c891a189008da8e78_d51381d83515f3cce487395d9d692757 store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:36,896 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208ca945729dbbe407c891a189008da8e78_d51381d83515f3cce487395d9d692757, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:36,896 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ca945729dbbe407c891a189008da8e78_d51381d83515f3cce487395d9d692757 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:36,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742333_1509 (size=12207) 2024-12-08T03:42:36,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742332_1508 (size=4469) 2024-12-08T03:42:36,903 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/41e12b3898b2443b90fb8f54ba2d1cc1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/41e12b3898b2443b90fb8f54ba2d1cc1 2024-12-08T03:42:36,906 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/B of d51381d83515f3cce487395d9d692757 into 41e12b3898b2443b90fb8f54ba2d1cc1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:36,906 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:36,906 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/B, priority=13, startTime=1733629356888; duration=0sec 2024-12-08T03:42:36,907 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:36,907 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:B 2024-12-08T03:42:36,907 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:36,907 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:36,907 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/C is initiating minor compaction (all files) 2024-12-08T03:42:36,908 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/C in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,908 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1ab6879254314b3f8683ba5b8ab795ca, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/947671ba0df34c0ca20b6d4c311b645b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/254ac1523c0744aa97e25827982fbcc6] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.3 K 2024-12-08T03:42:36,908 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab6879254314b3f8683ba5b8ab795ca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733629350249 2024-12-08T03:42:36,908 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 947671ba0df34c0ca20b6d4c311b645b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733629352431 2024-12-08T03:42:36,908 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 254ac1523c0744aa97e25827982fbcc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733629354602 2024-12-08T03:42:36,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T03:42:36,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:36,916 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-08T03:42:36,919 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#C#compaction#434 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:36,919 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/db41ec64d9e044109c3b0202f6e9c5da is 50, key is test_row_0/C:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081c12409e1afc4ac28ac084b70e0eea5a_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629355287/Put/seqid=0 2024-12-08T03:42:36,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742334_1510 (size=12207) 2024-12-08T03:42:36,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742335_1511 (size=12154) 2024-12-08T03:42:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:36,931 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081c12409e1afc4ac28ac084b70e0eea5a_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081c12409e1afc4ac28ac084b70e0eea5a_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:36,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/31714a7a01ed48e88c0dbdc3ecc1f5e7, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:36,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/31714a7a01ed48e88c0dbdc3ecc1f5e7 is 175, key is test_row_0/A:col10/1733629355287/Put/seqid=0 2024-12-08T03:42:36,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742336_1512 (size=30955) 2024-12-08T03:42:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:37,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:37,301 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#A#compaction#433 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:37,302 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/b6dcfd0ea93841d7894988fdf2f7381c is 175, key is test_row_0/A:col10/1733629355240/Put/seqid=0 2024-12-08T03:42:37,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742337_1513 (size=31161) 2024-12-08T03:42:37,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629417319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,326 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/db41ec64d9e044109c3b0202f6e9c5da as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/db41ec64d9e044109c3b0202f6e9c5da 2024-12-08T03:42:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629417322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629417323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,330 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/C of d51381d83515f3cce487395d9d692757 into db41ec64d9e044109c3b0202f6e9c5da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:37,330 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:37,330 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/C, priority=13, startTime=1733629356888; duration=0sec 2024-12-08T03:42:37,330 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:37,330 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:C 2024-12-08T03:42:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,351 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/31714a7a01ed48e88c0dbdc3ecc1f5e7 2024-12-08T03:42:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/e31c22b4cff24e009932996d5fe399a4 is 50, key is test_row_0/B:col10/1733629355287/Put/seqid=0 2024-12-08T03:42:37,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742338_1514 (size=12001) 2024-12-08T03:42:37,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629417428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629417438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629417438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,440 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_0' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629417438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629417448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,451 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:37,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629417638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629417648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629417649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,712 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/b6dcfd0ea93841d7894988fdf2f7381c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/b6dcfd0ea93841d7894988fdf2f7381c 2024-12-08T03:42:37,716 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/A of d51381d83515f3cce487395d9d692757 into b6dcfd0ea93841d7894988fdf2f7381c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:37,716 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:37,716 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/A, priority=13, startTime=1733629356888; duration=0sec 2024-12-08T03:42:37,717 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:37,717 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:A 2024-12-08T03:42:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/e31c22b4cff24e009932996d5fe399a4 2024-12-08T03:42:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/56bdadc88b4246beb413ad6c51ffe534 is 50, key is test_row_0/C:col10/1733629355287/Put/seqid=0 2024-12-08T03:42:37,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742339_1515 (size=12001) 2024-12-08T03:42:37,775 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/56bdadc88b4246beb413ad6c51ffe534 2024-12-08T03:42:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/31714a7a01ed48e88c0dbdc3ecc1f5e7 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7 2024-12-08T03:42:37,783 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7, entries=150, sequenceid=118, filesize=30.2 K 2024-12-08T03:42:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/e31c22b4cff24e009932996d5fe399a4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/e31c22b4cff24e009932996d5fe399a4 2024-12-08T03:42:37,787 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/e31c22b4cff24e009932996d5fe399a4, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T03:42:37,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/56bdadc88b4246beb413ad6c51ffe534 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/56bdadc88b4246beb413ad6c51ffe534 2024-12-08T03:42:37,792 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/56bdadc88b4246beb413ad6c51ffe534, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T03:42:37,793 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for d51381d83515f3cce487395d9d692757 in 877ms, sequenceid=118, compaction requested=false 2024-12-08T03:42:37,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:37,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:37,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-08T03:42:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-08T03:42:37,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-08T03:42:37,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4040 sec 2024-12-08T03:42:37,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4080 sec 2024-12-08T03:42:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:37,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:37,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-08T03:42:37,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208878cd35fe6bd423ea9d0fa262cd07ae9_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:37,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742340_1516 (size=14744) 2024-12-08T03:42:37,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629417976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629417979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629417979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629418088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629418088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629418088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629418298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629418298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629418298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:38,363 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208878cd35fe6bd423ea9d0fa262cd07ae9_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208878cd35fe6bd423ea9d0fa262cd07ae9_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:38,363 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/9607f833a1e84041ad866ad3c12b9269, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:38,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/9607f833a1e84041ad866ad3c12b9269 is 175, key is test_row_0/A:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742341_1517 (size=39699) 2024-12-08T03:42:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629418608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629418608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629418608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:38,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/9607f833a1e84041ad866ad3c12b9269 2024-12-08T03:42:38,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ed2b699572114b7ea90cb04dc0ca291d is 50, key is test_row_0/B:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:38,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742342_1518 (size=12101) 2024-12-08T03:42:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629419118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629419118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629419118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:39,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ed2b699572114b7ea90cb04dc0ca291d 2024-12-08T03:42:39,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/23210f0b0fca4c55b232ba3d15de5392 is 50, key is test_row_0/C:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:39,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742343_1519 (size=12101) 2024-12-08T03:42:39,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629419458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:39,460 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_0' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:39,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629419468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:39,470 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T03:42:39,558 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:39,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-08T03:42:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T03:42:39,560 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:39,560 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:39,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:39,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/23210f0b0fca4c55b232ba3d15de5392 2024-12-08T03:42:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/9607f833a1e84041ad866ad3c12b9269 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269 2024-12-08T03:42:39,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269, entries=200, sequenceid=136, filesize=38.8 K 2024-12-08T03:42:39,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ed2b699572114b7ea90cb04dc0ca291d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ed2b699572114b7ea90cb04dc0ca291d 2024-12-08T03:42:39,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ed2b699572114b7ea90cb04dc0ca291d, entries=150, sequenceid=136, filesize=11.8 K 2024-12-08T03:42:39,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/23210f0b0fca4c55b232ba3d15de5392 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/23210f0b0fca4c55b232ba3d15de5392 2024-12-08T03:42:39,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/23210f0b0fca4c55b232ba3d15de5392, entries=150, sequenceid=136, filesize=11.8 K 2024-12-08T03:42:39,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d51381d83515f3cce487395d9d692757 in 1648ms, sequenceid=136, compaction requested=true 2024-12-08T03:42:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:39,599 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:39,599 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:39,599 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:39,599 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:39,599 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/A is initiating minor compaction (all files) 2024-12-08T03:42:39,599 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/B is initiating minor compaction (all files) 2024-12-08T03:42:39,599 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/A in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:39,599 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/B in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:39,600 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/41e12b3898b2443b90fb8f54ba2d1cc1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/e31c22b4cff24e009932996d5fe399a4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ed2b699572114b7ea90cb04dc0ca291d] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.5 K 2024-12-08T03:42:39,600 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/b6dcfd0ea93841d7894988fdf2f7381c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=99.4 K 2024-12-08T03:42:39,600 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/b6dcfd0ea93841d7894988fdf2f7381c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269] 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6dcfd0ea93841d7894988fdf2f7381c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733629354602 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 41e12b3898b2443b90fb8f54ba2d1cc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733629354602 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting e31c22b4cff24e009932996d5fe399a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629355286 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31714a7a01ed48e88c0dbdc3ecc1f5e7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629355286 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ed2b699572114b7ea90cb04dc0ca291d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733629357313 2024-12-08T03:42:39,600 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9607f833a1e84041ad866ad3c12b9269, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733629357313 2024-12-08T03:42:39,609 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:39,610 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/bc411ca140bc46bbbd30093b5b9a3a75 is 50, key is test_row_0/B:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:39,611 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:39,612 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120847319b1cfb8d413b9a8680af5703560c_d51381d83515f3cce487395d9d692757 store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:39,613 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120847319b1cfb8d413b9a8680af5703560c_d51381d83515f3cce487395d9d692757, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:39,613 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120847319b1cfb8d413b9a8680af5703560c_d51381d83515f3cce487395d9d692757 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742344_1520 (size=12409) 2024-12-08T03:42:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742345_1521 (size=4469) 2024-12-08T03:42:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T03:42:39,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T03:42:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:39,712 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-08T03:42:39,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084776e7fdaefe4391a95d5c3b665a35d0_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629357973/Put/seqid=0 2024-12-08T03:42:39,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742346_1522 (size=12304) 2024-12-08T03:42:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T03:42:40,018 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/bc411ca140bc46bbbd30093b5b9a3a75 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/bc411ca140bc46bbbd30093b5b9a3a75 2024-12-08T03:42:40,022 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#A#compaction#442 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:40,023 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/4fbc5c0eeba9416a8c7bffd0b2f4553d is 175, key is test_row_0/A:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:40,025 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/B of d51381d83515f3cce487395d9d692757 into bc411ca140bc46bbbd30093b5b9a3a75(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:40,025 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:40,025 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/B, priority=13, startTime=1733629359599; duration=0sec 2024-12-08T03:42:40,025 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:40,025 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:B 2024-12-08T03:42:40,025 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742347_1523 (size=31363) 2024-12-08T03:42:40,026 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:40,026 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/C is initiating minor compaction (all files) 2024-12-08T03:42:40,026 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/C in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:40,026 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/db41ec64d9e044109c3b0202f6e9c5da, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/56bdadc88b4246beb413ad6c51ffe534, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/23210f0b0fca4c55b232ba3d15de5392] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.5 K 2024-12-08T03:42:40,026 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting db41ec64d9e044109c3b0202f6e9c5da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733629354602 2024-12-08T03:42:40,026 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 56bdadc88b4246beb413ad6c51ffe534, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733629355286 2024-12-08T03:42:40,027 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 23210f0b0fca4c55b232ba3d15de5392, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733629357313 2024-12-08T03:42:40,030 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/4fbc5c0eeba9416a8c7bffd0b2f4553d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/4fbc5c0eeba9416a8c7bffd0b2f4553d 2024-12-08T03:42:40,034 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/A of d51381d83515f3cce487395d9d692757 into 4fbc5c0eeba9416a8c7bffd0b2f4553d(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:40,034 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:40,034 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/A, priority=13, startTime=1733629359598; duration=0sec 2024-12-08T03:42:40,034 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:40,034 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:A 2024-12-08T03:42:40,040 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:40,041 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/5232074511b44d8580798ba283096fee is 50, key is test_row_0/C:col10/1733629357313/Put/seqid=0 2024-12-08T03:42:40,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742348_1524 (size=12409) 2024-12-08T03:42:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:40,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:40,133 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084776e7fdaefe4391a95d5c3b665a35d0_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084776e7fdaefe4391a95d5c3b665a35d0_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:40,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/03cba63dd4e64df1a94411e65e8afc59, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:40,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/03cba63dd4e64df1a94411e65e8afc59 is 175, key is test_row_0/A:col10/1733629357973/Put/seqid=0 2024-12-08T03:42:40,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742349_1525 (size=31105) 2024-12-08T03:42:40,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629420148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629420148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629420151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T03:42:40,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629420258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629420258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629420258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,456 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/5232074511b44d8580798ba283096fee as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/5232074511b44d8580798ba283096fee 2024-12-08T03:42:40,460 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/C of d51381d83515f3cce487395d9d692757 into 5232074511b44d8580798ba283096fee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:40,460 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:40,460 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/C, priority=13, startTime=1733629359599; duration=0sec 2024-12-08T03:42:40,460 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:40,460 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:C 2024-12-08T03:42:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,539 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/03cba63dd4e64df1a94411e65e8afc59 2024-12-08T03:42:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/a2338fb6508a421a8dcc61d91e93e85e is 50, key is test_row_0/B:col10/1733629357973/Put/seqid=0 2024-12-08T03:42:40,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742350_1526 (size=12151) 2024-12-08T03:42:40,559 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/a2338fb6508a421a8dcc61d91e93e85e 2024-12-08T03:42:40,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/3aa8b2a6f2534279b43d17f814851b04 is 50, key is test_row_0/C:col10/1733629357973/Put/seqid=0 2024-12-08T03:42:40,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742351_1527 (size=12151) 2024-12-08T03:42:40,577 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/3aa8b2a6f2534279b43d17f814851b04 2024-12-08T03:42:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/03cba63dd4e64df1a94411e65e8afc59 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59 2024-12-08T03:42:40,584 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59, entries=150, sequenceid=156, filesize=30.4 K 2024-12-08T03:42:40,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/a2338fb6508a421a8dcc61d91e93e85e as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a2338fb6508a421a8dcc61d91e93e85e 2024-12-08T03:42:40,588 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a2338fb6508a421a8dcc61d91e93e85e, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T03:42:40,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/3aa8b2a6f2534279b43d17f814851b04 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/3aa8b2a6f2534279b43d17f814851b04 2024-12-08T03:42:40,592 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/3aa8b2a6f2534279b43d17f814851b04, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T03:42:40,593 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d51381d83515f3cce487395d9d692757 in 881ms, sequenceid=156, compaction requested=false 2024-12-08T03:42:40,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:40,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:40,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-08T03:42:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-08T03:42:40,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-08T03:42:40,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-12-08T03:42:40,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0390 sec 2024-12-08T03:42:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T03:42:40,698 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:40,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-08T03:42:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:40,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:40,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:40,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:40,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:40,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-08T03:42:40,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b90fe3e953de4a0981c8353659c06435_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:40,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742352_1528 (size=17284) 2024-12-08T03:42:40,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:40,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b90fe3e953de4a0981c8353659c06435_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b90fe3e953de4a0981c8353659c06435_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:40,797 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/da1bc21776e644a696890c9449e3aa2b, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:40,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/da1bc21776e644a696890c9449e3aa2b is 175, key is test_row_0/A:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:40,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742353_1529 (size=48389) 2024-12-08T03:42:40,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629420804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629420804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:40,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629420918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629420918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629420918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629421128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629421128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629421128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,200 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/da1bc21776e644a696890c9449e3aa2b 2024-12-08T03:42:41,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/a54c52c7cbfa46a29868921b14730dbd is 50, key is test_row_0/B:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:41,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742354_1530 (size=12151) 2024-12-08T03:42:41,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:41,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629421438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629421439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629421439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/a54c52c7cbfa46a29868921b14730dbd 2024-12-08T03:42:41,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/a8150ecf54d6432195ddb63f87299988 is 50, key is test_row_0/C:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:41,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742355_1531 (size=12151) 2024-12-08T03:42:41,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:41,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:41,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629421948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629421948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629421948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:42,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/a8150ecf54d6432195ddb63f87299988 2024-12-08T03:42:42,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/da1bc21776e644a696890c9449e3aa2b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b 2024-12-08T03:42:42,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b, entries=250, sequenceid=178, filesize=47.3 K 2024-12-08T03:42:42,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/a54c52c7cbfa46a29868921b14730dbd as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a54c52c7cbfa46a29868921b14730dbd 2024-12-08T03:42:42,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a54c52c7cbfa46a29868921b14730dbd, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T03:42:42,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/a8150ecf54d6432195ddb63f87299988 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/a8150ecf54d6432195ddb63f87299988 2024-12-08T03:42:42,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/a8150ecf54d6432195ddb63f87299988, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T03:42:42,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d51381d83515f3cce487395d9d692757 in 1251ms, sequenceid=178, compaction requested=true 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:42,031 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:42,031 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/A is initiating minor compaction (all files) 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/B is initiating minor compaction (all files) 2024-12-08T03:42:42,032 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/A in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:42,032 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/B in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:42,032 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/4fbc5c0eeba9416a8c7bffd0b2f4553d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=108.3 K 2024-12-08T03:42:42,032 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/bc411ca140bc46bbbd30093b5b9a3a75, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a2338fb6508a421a8dcc61d91e93e85e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a54c52c7cbfa46a29868921b14730dbd] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.9 K 2024-12-08T03:42:42,032 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/4fbc5c0eeba9416a8c7bffd0b2f4553d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b] 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bc411ca140bc46bbbd30093b5b9a3a75, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733629357313 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbc5c0eeba9416a8c7bffd0b2f4553d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733629357313 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a2338fb6508a421a8dcc61d91e93e85e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733629357973 2024-12-08T03:42:42,032 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03cba63dd4e64df1a94411e65e8afc59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733629357973 2024-12-08T03:42:42,033 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a54c52c7cbfa46a29868921b14730dbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733629360779 2024-12-08T03:42:42,033 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1bc21776e644a696890c9449e3aa2b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733629360147 2024-12-08T03:42:42,036 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:42,037 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:42,037 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/7bf823fb9a41497fba439dcb7454c934 is 50, key is test_row_0/B:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:42,038 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208dfa81ef7aa6c4ae49dcd651771352498_d51381d83515f3cce487395d9d692757 store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:42,040 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208dfa81ef7aa6c4ae49dcd651771352498_d51381d83515f3cce487395d9d692757, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:42,040 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dfa81ef7aa6c4ae49dcd651771352498_d51381d83515f3cce487395d9d692757 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742356_1532 (size=12561) 2024-12-08T03:42:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742357_1533 (size=4469) 2024-12-08T03:42:42,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T03:42:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:42,074 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-08T03:42:42,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087515de1d9ca84094b04d1f9e65ea3bb6_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629360803/Put/seqid=0 2024-12-08T03:42:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742358_1534 (size=12304) 2024-12-08T03:42:42,445 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#A#compaction#451 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:42,445 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/0005f069f22347e999d788508ff096bc is 175, key is test_row_0/A:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:42,446 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/7bf823fb9a41497fba439dcb7454c934 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/7bf823fb9a41497fba439dcb7454c934 2024-12-08T03:42:42,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742359_1535 (size=31515) 2024-12-08T03:42:42,457 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/B of d51381d83515f3cce487395d9d692757 into 7bf823fb9a41497fba439dcb7454c934(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:42,457 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:42,457 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/B, priority=13, startTime=1733629362031; duration=0sec 2024-12-08T03:42:42,457 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/0005f069f22347e999d788508ff096bc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/0005f069f22347e999d788508ff096bc 2024-12-08T03:42:42,457 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:42,457 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:B 2024-12-08T03:42:42,457 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:42,458 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:42,458 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/C is initiating minor compaction (all files) 2024-12-08T03:42:42,458 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/C in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:42,458 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/5232074511b44d8580798ba283096fee, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/3aa8b2a6f2534279b43d17f814851b04, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/a8150ecf54d6432195ddb63f87299988] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=35.9 K 2024-12-08T03:42:42,458 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 5232074511b44d8580798ba283096fee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733629357313 2024-12-08T03:42:42,459 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aa8b2a6f2534279b43d17f814851b04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733629357973 2024-12-08T03:42:42,459 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting a8150ecf54d6432195ddb63f87299988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733629360779 2024-12-08T03:42:42,460 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/A of d51381d83515f3cce487395d9d692757 into 0005f069f22347e999d788508ff096bc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:42,461 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:42,461 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/A, priority=13, startTime=1733629362031; duration=0sec 2024-12-08T03:42:42,461 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:42,461 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:A 2024-12-08T03:42:42,464 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:42,464 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/79b9df35140f46a091f7e64abde5bea5 is 50, key is test_row_0/C:col10/1733629360780/Put/seqid=0 2024-12-08T03:42:42,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742360_1536 (size=12561) 2024-12-08T03:42:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:42,485 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087515de1d9ca84094b04d1f9e65ea3bb6_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087515de1d9ca84094b04d1f9e65ea3bb6_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/386656414f43401688ed8ff50e40f4db, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/386656414f43401688ed8ff50e40f4db is 175, key is test_row_0/A:col10/1733629360803/Put/seqid=0 2024-12-08T03:42:42,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742361_1537 (size=31105) 2024-12-08T03:42:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:42,871 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/79b9df35140f46a091f7e64abde5bea5 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/79b9df35140f46a091f7e64abde5bea5 2024-12-08T03:42:42,874 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/C of d51381d83515f3cce487395d9d692757 into 79b9df35140f46a091f7e64abde5bea5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:42,874 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:42,874 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/C, priority=13, startTime=1733629362031; duration=0sec 2024-12-08T03:42:42,874 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:42,874 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:C 2024-12-08T03:42:42,889 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/386656414f43401688ed8ff50e40f4db 2024-12-08T03:42:42,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/cf2e8937ed0f4fcdad395c48c40c566f is 50, key is test_row_0/B:col10/1733629360803/Put/seqid=0 2024-12-08T03:42:42,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742362_1538 (size=12151) 2024-12-08T03:42:42,900 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/cf2e8937ed0f4fcdad395c48c40c566f 2024-12-08T03:42:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/1f6b2ed66139426f818e247da735ccb2 is 50, key is test_row_0/C:col10/1733629360803/Put/seqid=0 2024-12-08T03:42:42,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742363_1539 (size=12151) 2024-12-08T03:42:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629422974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629422975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629422975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629423088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629423088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629423089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629423298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629423298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629423298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,310 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/1f6b2ed66139426f818e247da735ccb2 2024-12-08T03:42:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/386656414f43401688ed8ff50e40f4db as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db 2024-12-08T03:42:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db, entries=150, sequenceid=196, filesize=30.4 K 2024-12-08T03:42:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/cf2e8937ed0f4fcdad395c48c40c566f as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cf2e8937ed0f4fcdad395c48c40c566f 2024-12-08T03:42:43,321 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cf2e8937ed0f4fcdad395c48c40c566f, entries=150, sequenceid=196, filesize=11.9 K 2024-12-08T03:42:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/1f6b2ed66139426f818e247da735ccb2 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1f6b2ed66139426f818e247da735ccb2 2024-12-08T03:42:43,325 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1f6b2ed66139426f818e247da735ccb2, entries=150, sequenceid=196, filesize=11.9 K 2024-12-08T03:42:43,325 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d51381d83515f3cce487395d9d692757 in 1251ms, sequenceid=196, compaction requested=false 2024-12-08T03:42:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-08T03:42:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-08T03:42:43,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-08T03:42:43,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6260 sec 2024-12-08T03:42:43,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.6300 sec 2024-12-08T03:42:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:43,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:43,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-08T03:42:43,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ad5272fbc995417abf2aa8c31ec14910_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:43,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742364_1540 (size=14794) 2024-12-08T03:42:43,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629423520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629423521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629423608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629423608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629423609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629423628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629423628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629423839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629423839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:43,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ad5272fbc995417abf2aa8c31ec14910_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ad5272fbc995417abf2aa8c31ec14910_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:43,891 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/e50d8d941ad24c84b93312790fab3320, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:43,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/e50d8d941ad24c84b93312790fab3320 is 175, key is test_row_0/A:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:43,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742365_1541 (size=39749) 2024-12-08T03:42:44,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629424118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629424118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629424118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629424148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629424148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/e50d8d941ad24c84b93312790fab3320 2024-12-08T03:42:44,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/739915060b0c4a078a566fd4502b4104 is 50, key is test_row_0/B:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:44,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742366_1542 (size=12151) 2024-12-08T03:42:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629424658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629424658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:44,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/739915060b0c4a078a566fd4502b4104 2024-12-08T03:42:44,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d61d953aac9349a7b1980f69d1a39378 is 50, key is test_row_0/C:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742367_1543 (size=12151) 2024-12-08T03:42:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T03:42:44,868 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:44,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-08T03:42:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:44,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:44,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:44,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:45,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d61d953aac9349a7b1980f69d1a39378 2024-12-08T03:42:45,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/e50d8d941ad24c84b93312790fab3320 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320 2024-12-08T03:42:45,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320, entries=200, sequenceid=218, filesize=38.8 K 2024-12-08T03:42:45,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/739915060b0c4a078a566fd4502b4104 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/739915060b0c4a078a566fd4502b4104 2024-12-08T03:42:45,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/739915060b0c4a078a566fd4502b4104, entries=150, sequenceid=218, filesize=11.9 K 2024-12-08T03:42:45,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d61d953aac9349a7b1980f69d1a39378 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d61d953aac9349a7b1980f69d1a39378 2024-12-08T03:42:45,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d61d953aac9349a7b1980f69d1a39378, entries=150, sequenceid=218, filesize=11.9 K 2024-12-08T03:42:45,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d51381d83515f3cce487395d9d692757 in 1644ms, sequenceid=218, compaction requested=true 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:45,124 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:45,124 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:45,124 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:45,124 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:45,124 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/A is initiating minor compaction (all files) 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/B is initiating minor compaction (all files) 2024-12-08T03:42:45,125 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/A in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,125 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/B in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,125 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/0005f069f22347e999d788508ff096bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=100.0 K 2024-12-08T03:42:45,125 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/7bf823fb9a41497fba439dcb7454c934, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cf2e8937ed0f4fcdad395c48c40c566f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/739915060b0c4a078a566fd4502b4104] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=36.0 K 2024-12-08T03:42:45,125 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/0005f069f22347e999d788508ff096bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320] 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf823fb9a41497fba439dcb7454c934, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733629360779 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0005f069f22347e999d788508ff096bc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733629360779 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2e8937ed0f4fcdad395c48c40c566f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733629360796 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386656414f43401688ed8ff50e40f4db, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733629360796 2024-12-08T03:42:45,125 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 739915060b0c4a078a566fd4502b4104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733629362971 2024-12-08T03:42:45,126 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting e50d8d941ad24c84b93312790fab3320, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733629362971 2024-12-08T03:42:45,130 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:45,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-08T03:42:45,131 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:45,131 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/0dcc065db9524c1bbfacd6310b1431d4 is 50, key is test_row_0/B:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:45,132 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120888502ea0c7be49469e794c965baccc55_d51381d83515f3cce487395d9d692757 store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:45,133 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120888502ea0c7be49469e794c965baccc55_d51381d83515f3cce487395d9d692757, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:45,134 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120888502ea0c7be49469e794c965baccc55_d51381d83515f3cce487395d9d692757 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742368_1544 (size=12663) 2024-12-08T03:42:45,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120811a0229edb4f4b2bb04ce43b3916ee3d_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629365130/Put/seqid=0 2024-12-08T03:42:45,139 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/0dcc065db9524c1bbfacd6310b1431d4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/0dcc065db9524c1bbfacd6310b1431d4 2024-12-08T03:42:45,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742369_1545 (size=4469) 2024-12-08T03:42:45,144 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/B of d51381d83515f3cce487395d9d692757 into 0dcc065db9524c1bbfacd6310b1431d4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:45,144 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:45,144 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/B, priority=13, startTime=1733629365124; duration=0sec 2024-12-08T03:42:45,144 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:45,144 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:B 2024-12-08T03:42:45,144 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:42:45,145 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:42:45,145 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/C is initiating minor compaction (all files) 2024-12-08T03:42:45,145 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/C in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,145 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/79b9df35140f46a091f7e64abde5bea5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1f6b2ed66139426f818e247da735ccb2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d61d953aac9349a7b1980f69d1a39378] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=36.0 K 2024-12-08T03:42:45,145 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b9df35140f46a091f7e64abde5bea5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733629360779 2024-12-08T03:42:45,146 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6b2ed66139426f818e247da735ccb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733629360796 2024-12-08T03:42:45,146 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#A#compaction#460 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:45,146 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting d61d953aac9349a7b1980f69d1a39378, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733629362971 2024-12-08T03:42:45,146 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/5afeb18bd42841339ec440354957a7bc is 175, key is test_row_0/A:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:45,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742370_1546 (size=17284) 2024-12-08T03:42:45,148 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:45,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120811a0229edb4f4b2bb04ce43b3916ee3d_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120811a0229edb4f4b2bb04ce43b3916ee3d_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:45,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/80b726f8e752436cabddc3f835c01a16, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:45,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/80b726f8e752436cabddc3f835c01a16 is 175, key is test_row_0/A:col10/1733629365130/Put/seqid=0 2024-12-08T03:42:45,153 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:45,153 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/bb74d449bb5444a1ac44c14087cf6f9b is 50, key is test_row_0/C:col10/1733629363478/Put/seqid=0 2024-12-08T03:42:45,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742371_1547 (size=31617) 2024-12-08T03:42:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629425155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629425156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629425157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,164 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/5afeb18bd42841339ec440354957a7bc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/5afeb18bd42841339ec440354957a7bc 2024-12-08T03:42:45,168 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/A of d51381d83515f3cce487395d9d692757 into 5afeb18bd42841339ec440354957a7bc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:45,168 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:45,168 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/A, priority=13, startTime=1733629365124; duration=0sec 2024-12-08T03:42:45,168 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:45,168 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:A 2024-12-08T03:42:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742373_1549 (size=12663) 2024-12-08T03:42:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742372_1548 (size=48389) 2024-12-08T03:42:45,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/80b726f8e752436cabddc3f835c01a16 2024-12-08T03:42:45,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ae07fb45bf094841816eda112dd46e2d is 50, key is test_row_0/B:col10/1733629365130/Put/seqid=0 2024-12-08T03:42:45,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742374_1550 (size=12151) 2024-12-08T03:42:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:45,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629425268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629425268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629425268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629425478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629425478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629425478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:45,576 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/bb74d449bb5444a1ac44c14087cf6f9b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/bb74d449bb5444a1ac44c14087cf6f9b 2024-12-08T03:42:45,579 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51381d83515f3cce487395d9d692757/C of d51381d83515f3cce487395d9d692757 into bb74d449bb5444a1ac44c14087cf6f9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:45,579 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:45,579 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/C, priority=13, startTime=1733629365124; duration=0sec 2024-12-08T03:42:45,579 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:45,579 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:C 2024-12-08T03:42:45,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ae07fb45bf094841816eda112dd46e2d 2024-12-08T03:42:45,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/61fa52502bd24400907d4469f3f02d98 is 50, key is test_row_0/C:col10/1733629365130/Put/seqid=0 2024-12-08T03:42:45,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742375_1551 (size=12151) 2024-12-08T03:42:45,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629425668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,669 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629425678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,680 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629425788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629425788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629425788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:45,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:45,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:42:45,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/61fa52502bd24400907d4469f3f02d98 2024-12-08T03:42:45,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/80b726f8e752436cabddc3f835c01a16 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16 2024-12-08T03:42:45,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16, entries=250, sequenceid=238, filesize=47.3 K 2024-12-08T03:42:45,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ae07fb45bf094841816eda112dd46e2d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ae07fb45bf094841816eda112dd46e2d 2024-12-08T03:42:46,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ae07fb45bf094841816eda112dd46e2d, entries=150, sequenceid=238, filesize=11.9 K 2024-12-08T03:42:46,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/61fa52502bd24400907d4469f3f02d98 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/61fa52502bd24400907d4469f3f02d98 2024-12-08T03:42:46,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/61fa52502bd24400907d4469f3f02d98, entries=150, sequenceid=238, filesize=11.9 K 2024-12-08T03:42:46,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d51381d83515f3cce487395d9d692757 in 874ms, sequenceid=238, compaction requested=false 2024-12-08T03:42:46,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:46,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T03:42:46,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:46,091 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-08T03:42:46,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f868eb5ca8904da0be3ef68db139e1b6_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629365155/Put/seqid=0 2024-12-08T03:42:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742376_1552 (size=12304) 2024-12-08T03:42:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:46,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629426318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629426321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629426321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629426428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629426428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629426429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:46,503 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f868eb5ca8904da0be3ef68db139e1b6_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f868eb5ca8904da0be3ef68db139e1b6_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f745bdb3a56445b98d92aab3a298d8fc, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f745bdb3a56445b98d92aab3a298d8fc is 175, key is test_row_0/A:col10/1733629365155/Put/seqid=0 2024-12-08T03:42:46,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742377_1553 (size=31105) 2024-12-08T03:42:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629426638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629426638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629426638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,907 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f745bdb3a56445b98d92aab3a298d8fc 2024-12-08T03:42:46,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/106c231a876a4f9ebc2a956296c4af30 is 50, key is test_row_0/B:col10/1733629365155/Put/seqid=0 2024-12-08T03:42:46,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742378_1554 (size=12151) 2024-12-08T03:42:46,915 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/106c231a876a4f9ebc2a956296c4af30 2024-12-08T03:42:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/566b575020d242f28b4e27e2f4359b12 is 50, key is test_row_0/C:col10/1733629365155/Put/seqid=0 2024-12-08T03:42:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742379_1555 (size=12151) 2024-12-08T03:42:46,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629426948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629426948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629426948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:47,323 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/566b575020d242f28b4e27e2f4359b12 2024-12-08T03:42:47,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f745bdb3a56445b98d92aab3a298d8fc as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc 2024-12-08T03:42:47,329 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc, entries=150, sequenceid=257, filesize=30.4 K 2024-12-08T03:42:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/106c231a876a4f9ebc2a956296c4af30 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/106c231a876a4f9ebc2a956296c4af30 2024-12-08T03:42:47,332 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/106c231a876a4f9ebc2a956296c4af30, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T03:42:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/566b575020d242f28b4e27e2f4359b12 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/566b575020d242f28b4e27e2f4359b12 2024-12-08T03:42:47,335 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/566b575020d242f28b4e27e2f4359b12, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T03:42:47,335 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for d51381d83515f3cce487395d9d692757 in 1245ms, sequenceid=257, compaction requested=true 2024-12-08T03:42:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-08T03:42:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-08T03:42:47,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-08T03:42:47,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4670 sec 2024-12-08T03:42:47,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.4700 sec 2024-12-08T03:42:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-08T03:42:47,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-08T03:42:47,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208817ce0e8f38a41a4b032b5f27dddc1de_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:47,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742380_1556 (size=17534) 2024-12-08T03:42:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629427476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629427477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629427478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629427588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629427588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629427588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38526 deadline: 1733629427688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,689 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_1' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38484 deadline: 1733629427708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,710 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,42629,1733629194165 for 'test_row_2' in d51381d83515f3cce487395d9d692757 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:42:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629427798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629427798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629427798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:47,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:47,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208817ce0e8f38a41a4b032b5f27dddc1de_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208817ce0e8f38a41a4b032b5f27dddc1de_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:47,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/34470c31b5f544a188321d0395d9b87a, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:47,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/34470c31b5f544a188321d0395d9b87a is 175, key is test_row_0/A:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:47,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742381_1557 (size=48639) 2024-12-08T03:42:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629428112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629428112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:48,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629428112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:48,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/34470c31b5f544a188321d0395d9b87a 2024-12-08T03:42:48,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ea9e53c655db4edfa61027951588cea1 is 50, key is test_row_0/B:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742382_1558 (size=12301) 2024-12-08T03:42:48,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1733629428618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:48,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38486 deadline: 1733629428618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38498 deadline: 1733629428618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 , the old value is region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., hostname=32df4b4bb439,42629,1733629194165, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:42:48,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ea9e53c655db4edfa61027951588cea1 2024-12-08T03:42:48,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/7a10d83f0fc540829c3a1674b6990d74 is 50, key is test_row_0/C:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:48,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742383_1559 (size=12301) 2024-12-08T03:42:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T03:42:49,028 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:49,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:42:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-08T03:42:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T03:42:49,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:42:49,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:42:49,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:42:49,074 INFO [Thread-2199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2199. 2024-12-08T03:42:49,074 DEBUG [Thread-2199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,074 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,074 INFO [Thread-2195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2195. 2024-12-08T03:42:49,074 DEBUG [Thread-2195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,075 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,075 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,075 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,076 INFO [Thread-2197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2197. 2024-12-08T03:42:49,076 DEBUG [Thread-2197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,076 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,076 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,078 INFO [Thread-2191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2191. 2024-12-08T03:42:49,078 DEBUG [Thread-2191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,078 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,078 INFO [Thread-2193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2193. 2024-12-08T03:42:49,078 DEBUG [Thread-2193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,078 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,078 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,078 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/7a10d83f0fc540829c3a1674b6990d74 2024-12-08T03:42:49,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/34470c31b5f544a188321d0395d9b87a as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a 2024-12-08T03:42:49,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a, entries=250, sequenceid=281, filesize=47.5 K 2024-12-08T03:42:49,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/ea9e53c655db4edfa61027951588cea1 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ea9e53c655db4edfa61027951588cea1 2024-12-08T03:42:49,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ea9e53c655db4edfa61027951588cea1, entries=150, sequenceid=281, filesize=12.0 K 2024-12-08T03:42:49,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/7a10d83f0fc540829c3a1674b6990d74 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/7a10d83f0fc540829c3a1674b6990d74 2024-12-08T03:42:49,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/7a10d83f0fc540829c3a1674b6990d74, entries=150, sequenceid=281, filesize=12.0 K 2024-12-08T03:42:49,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d51381d83515f3cce487395d9d692757 in 1677ms, sequenceid=281, compaction requested=true 2024-12-08T03:42:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:42:49,138 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51381d83515f3cce487395d9d692757:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:42:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:49,138 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:42:49,139 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:42:49,139 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:42:49,139 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/B is initiating minor compaction (all files) 2024-12-08T03:42:49,139 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/A is initiating minor compaction (all files) 2024-12-08T03:42:49,139 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/B in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:49,139 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/A in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:49,140 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/0dcc065db9524c1bbfacd6310b1431d4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ae07fb45bf094841816eda112dd46e2d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/106c231a876a4f9ebc2a956296c4af30, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ea9e53c655db4edfa61027951588cea1] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=48.1 K 2024-12-08T03:42:49,140 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/5afeb18bd42841339ec440354957a7bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=156.0 K 2024-12-08T03:42:49,140 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:49,140 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. files: [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/5afeb18bd42841339ec440354957a7bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a] 2024-12-08T03:42:49,140 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dcc065db9524c1bbfacd6310b1431d4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733629362971 2024-12-08T03:42:49,140 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5afeb18bd42841339ec440354957a7bc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733629362971 2024-12-08T03:42:49,140 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ae07fb45bf094841816eda112dd46e2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629363515 2024-12-08T03:42:49,140 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b726f8e752436cabddc3f835c01a16, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629363515 2024-12-08T03:42:49,140 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 106c231a876a4f9ebc2a956296c4af30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733629365137 2024-12-08T03:42:49,141 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting f745bdb3a56445b98d92aab3a298d8fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733629365137 2024-12-08T03:42:49,141 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9e53c655db4edfa61027951588cea1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733629367460 2024-12-08T03:42:49,141 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34470c31b5f544a188321d0395d9b87a, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733629366316 2024-12-08T03:42:49,149 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:49,150 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/321ea8ae048245b1b7c6365bb0440eee is 50, key is test_row_0/B:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:49,151 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:49,155 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208beec5cbe895247598e7522ead8efe079_d51381d83515f3cce487395d9d692757 store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:49,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742384_1560 (size=12949) 2024-12-08T03:42:49,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T03:42:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:49,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-08T03:42:49,191 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208beec5cbe895247598e7522ead8efe079_d51381d83515f3cce487395d9d692757, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:49,191 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208beec5cbe895247598e7522ead8efe079_d51381d83515f3cce487395d9d692757 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a3a8214a9dfd453c851365c621b84f8b_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629367472/Put/seqid=0 2024-12-08T03:42:49,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742385_1561 (size=4469) 2024-12-08T03:42:49,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742386_1562 (size=12454) 2024-12-08T03:42:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T03:42:49,562 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/321ea8ae048245b1b7c6365bb0440eee as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/321ea8ae048245b1b7c6365bb0440eee 2024-12-08T03:42:49,567 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51381d83515f3cce487395d9d692757/B of d51381d83515f3cce487395d9d692757 into 321ea8ae048245b1b7c6365bb0440eee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:49,567 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:49,568 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/B, priority=12, startTime=1733629369138; duration=0sec 2024-12-08T03:42:49,568 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:42:49,568 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:B 2024-12-08T03:42:49,568 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:42:49,570 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:42:49,570 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1541): d51381d83515f3cce487395d9d692757/C is initiating minor compaction (all files) 2024-12-08T03:42:49,570 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51381d83515f3cce487395d9d692757/C in TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:49,570 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/bb74d449bb5444a1ac44c14087cf6f9b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/61fa52502bd24400907d4469f3f02d98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/566b575020d242f28b4e27e2f4359b12, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/7a10d83f0fc540829c3a1674b6990d74] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp, totalSize=48.1 K 2024-12-08T03:42:49,570 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting bb74d449bb5444a1ac44c14087cf6f9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733629362971 2024-12-08T03:42:49,571 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fa52502bd24400907d4469f3f02d98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733629363515 2024-12-08T03:42:49,571 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 566b575020d242f28b4e27e2f4359b12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733629365137 2024-12-08T03:42:49,572 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a10d83f0fc540829c3a1674b6990d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733629367460 2024-12-08T03:42:49,583 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:49,583 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d5e9b6ffbc7245e386f7ed424df26933 is 50, key is test_row_0/C:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:49,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742387_1563 (size=12949) 2024-12-08T03:42:49,595 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51381d83515f3cce487395d9d692757#A#compaction#472 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:42:49,596 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/39c8eff9b8f449948bc9bf626f7e5228 is 175, key is test_row_0/A:col10/1733629367461/Put/seqid=0 2024-12-08T03:42:49,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742388_1564 (size=31903) 2024-12-08T03:42:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:49,603 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a3a8214a9dfd453c851365c621b84f8b_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a3a8214a9dfd453c851365c621b84f8b_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:49,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/7667b18ca5414b30a867b297f810760b, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:49,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/7667b18ca5414b30a867b297f810760b is 175, key is test_row_0/A:col10/1733629367472/Put/seqid=0 2024-12-08T03:42:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742389_1565 (size=31255) 2024-12-08T03:42:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42629 {}] regionserver.HRegion(8855): Flush requested on d51381d83515f3cce487395d9d692757 2024-12-08T03:42:49,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. as already flushing 2024-12-08T03:42:49,630 INFO [Thread-2186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2186. 2024-12-08T03:42:49,630 INFO [Thread-2184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2184. 2024-12-08T03:42:49,630 DEBUG [Thread-2184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,630 DEBUG [Thread-2186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,630 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,630 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,630 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,630 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,639 INFO [Thread-2180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2180. 2024-12-08T03:42:49,639 DEBUG [Thread-2180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:49,639 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,639 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:49,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T03:42:49,992 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/d5e9b6ffbc7245e386f7ed424df26933 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d5e9b6ffbc7245e386f7ed424df26933 2024-12-08T03:42:49,997 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51381d83515f3cce487395d9d692757/C of d51381d83515f3cce487395d9d692757 into d5e9b6ffbc7245e386f7ed424df26933(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:49,997 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:49,997 INFO [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/C, priority=12, startTime=1733629369138; duration=0sec 2024-12-08T03:42:49,997 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:49,997 DEBUG [RS:0;32df4b4bb439:42629-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:C 2024-12-08T03:42:50,003 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/39c8eff9b8f449948bc9bf626f7e5228 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/39c8eff9b8f449948bc9bf626f7e5228 2024-12-08T03:42:50,008 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/7667b18ca5414b30a867b297f810760b 2024-12-08T03:42:50,009 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51381d83515f3cce487395d9d692757/A of d51381d83515f3cce487395d9d692757 into 39c8eff9b8f449948bc9bf626f7e5228(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:42:50,009 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:50,009 INFO [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., storeName=d51381d83515f3cce487395d9d692757/A, priority=12, startTime=1733629369138; duration=0sec 2024-12-08T03:42:50,009 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:42:50,009 DEBUG [RS:0;32df4b4bb439:42629-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51381d83515f3cce487395d9d692757:A 2024-12-08T03:42:50,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/4e882d960daa463f9405811faf40e32c is 50, key is test_row_0/B:col10/1733629367472/Put/seqid=0 2024-12-08T03:42:50,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742390_1566 (size=12301) 2024-12-08T03:42:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T03:42:50,421 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/4e882d960daa463f9405811faf40e32c 2024-12-08T03:42:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/92dc79f07905415d8177c20cefdee212 is 50, key is test_row_0/C:col10/1733629367472/Put/seqid=0 2024-12-08T03:42:50,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742391_1567 (size=12301) 2024-12-08T03:42:50,837 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/92dc79f07905415d8177c20cefdee212 2024-12-08T03:42:50,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/7667b18ca5414b30a867b297f810760b as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/7667b18ca5414b30a867b297f810760b 2024-12-08T03:42:50,853 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/7667b18ca5414b30a867b297f810760b, entries=150, sequenceid=293, filesize=30.5 K 2024-12-08T03:42:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/4e882d960daa463f9405811faf40e32c as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/4e882d960daa463f9405811faf40e32c 2024-12-08T03:42:50,858 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/4e882d960daa463f9405811faf40e32c, entries=150, sequenceid=293, filesize=12.0 K 2024-12-08T03:42:50,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/92dc79f07905415d8177c20cefdee212 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/92dc79f07905415d8177c20cefdee212 2024-12-08T03:42:50,863 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/92dc79f07905415d8177c20cefdee212, entries=150, sequenceid=293, filesize=12.0 K 2024-12-08T03:42:50,864 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=20.13 KB/20610 for d51381d83515f3cce487395d9d692757 in 1682ms, sequenceid=293, compaction requested=false 2024-12-08T03:42:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for d51381d83515f3cce487395d9d692757: 2024-12-08T03:42:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-08T03:42:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-08T03:42:50,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-08T03:42:50,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8360 sec 2024-12-08T03:42:50,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8410 sec 2024-12-08T03:42:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T03:42:51,179 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:51,713 INFO [Thread-2182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2182. 2024-12-08T03:42:51,713 DEBUG [Thread-2182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:51,713 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:51,713 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:51,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:51,721 INFO [Thread-2188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2188. 2024-12-08T03:42:51,721 DEBUG [Thread-2188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:42:51,721 DEBUG [Thread-2188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:51,721 DEBUG [Thread-2188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:51,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:51,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4957 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4834 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4838 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4942 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4833 2024-12-08T03:42:51,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:42:51,722 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:42:51,722 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:42:51,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:51,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:51,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:51,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:42:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T03:42:51,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629371725"}]},"ts":"1733629371725"} 2024-12-08T03:42:51,727 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:42:51,727 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:42:51,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:42:51,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, UNASSIGN}] 2024-12-08T03:42:51,729 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, UNASSIGN 2024-12-08T03:42:51,730 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=CLOSING, regionLocation=32df4b4bb439,42629,1733629194165 2024-12-08T03:42:51,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, UNASSIGN because future has completed 2024-12-08T03:42:51,731 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:42:51,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165}] 2024-12-08T03:42:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T03:42:51,884 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close d51381d83515f3cce487395d9d692757 2024-12-08T03:42:51,884 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:42:51,885 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing d51381d83515f3cce487395d9d692757, disabling compactions & flushes 2024-12-08T03:42:51,885 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:51,885 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:51,885 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. after waiting 0 ms 2024-12-08T03:42:51,885 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:51,885 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing d51381d83515f3cce487395d9d692757 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-08T03:42:51,895 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088e98f863791344c6b140e4a591c7efb2_d51381d83515f3cce487395d9d692757 is 50, key is test_row_0/A:col10/1733629369638/Put/seqid=0 2024-12-08T03:42:51,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742392_1568 (size=12454) 2024-12-08T03:42:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T03:42:52,301 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:42:52,310 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088e98f863791344c6b140e4a591c7efb2_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088e98f863791344c6b140e4a591c7efb2_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:52,311 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f89722237d4e410992326586d7b070d6, store: [table=TestAcidGuarantees family=A region=d51381d83515f3cce487395d9d692757] 2024-12-08T03:42:52,312 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f89722237d4e410992326586d7b070d6 is 175, key is test_row_0/A:col10/1733629369638/Put/seqid=0 2024-12-08T03:42:52,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742393_1569 (size=31255) 2024-12-08T03:42:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T03:42:52,368 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:42:52,717 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f89722237d4e410992326586d7b070d6 2024-12-08T03:42:52,725 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/cffab11cfd6e494490a91d693d338d92 is 50, key is test_row_0/B:col10/1733629369638/Put/seqid=0 2024-12-08T03:42:52,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742394_1570 (size=12301) 2024-12-08T03:42:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T03:42:53,131 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/cffab11cfd6e494490a91d693d338d92 2024-12-08T03:42:53,145 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/954f2810138f4a25b01244912727a306 is 50, key is test_row_0/C:col10/1733629369638/Put/seqid=0 2024-12-08T03:42:53,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742395_1571 (size=12301) 2024-12-08T03:42:53,550 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/954f2810138f4a25b01244912727a306 2024-12-08T03:42:53,560 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/A/f89722237d4e410992326586d7b070d6 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f89722237d4e410992326586d7b070d6 2024-12-08T03:42:53,566 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f89722237d4e410992326586d7b070d6, entries=150, sequenceid=304, filesize=30.5 K 2024-12-08T03:42:53,567 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/B/cffab11cfd6e494490a91d693d338d92 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cffab11cfd6e494490a91d693d338d92 2024-12-08T03:42:53,571 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cffab11cfd6e494490a91d693d338d92, entries=150, sequenceid=304, filesize=12.0 K 2024-12-08T03:42:53,572 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/.tmp/C/954f2810138f4a25b01244912727a306 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/954f2810138f4a25b01244912727a306 2024-12-08T03:42:53,576 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/954f2810138f4a25b01244912727a306, entries=150, sequenceid=304, filesize=12.0 K 2024-12-08T03:42:53,577 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d51381d83515f3cce487395d9d692757 in 1692ms, sequenceid=304, compaction requested=true 2024-12-08T03:42:53,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/52a6a6f8c63d4d1593f20f10295816ca, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/b6dcfd0ea93841d7894988fdf2f7381c, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/4fbc5c0eeba9416a8c7bffd0b2f4553d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/0005f069f22347e999d788508ff096bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/5afeb18bd42841339ec440354957a7bc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a] to archive 2024-12-08T03:42:53,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:42:53,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/ac4d046506c14f6fbaf3630365638e17 2024-12-08T03:42:53,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f6c833d6ee3949449304bf12b6001f3c 2024-12-08T03:42:53,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/905bc9e1b87941ee9a991216811c48e0 2024-12-08T03:42:53,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/52a6a6f8c63d4d1593f20f10295816ca to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/52a6a6f8c63d4d1593f20f10295816ca 2024-12-08T03:42:53,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/a7897c2840c843fd9afcd46fef6cb440 2024-12-08T03:42:53,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/447212f64442448bb07b53d02413bb71 2024-12-08T03:42:53,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/b6dcfd0ea93841d7894988fdf2f7381c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/b6dcfd0ea93841d7894988fdf2f7381c 2024-12-08T03:42:53,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/31714a7a01ed48e88c0dbdc3ecc1f5e7 2024-12-08T03:42:53,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/9607f833a1e84041ad866ad3c12b9269 2024-12-08T03:42:53,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/4fbc5c0eeba9416a8c7bffd0b2f4553d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/4fbc5c0eeba9416a8c7bffd0b2f4553d 2024-12-08T03:42:53,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/03cba63dd4e64df1a94411e65e8afc59 2024-12-08T03:42:53,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/da1bc21776e644a696890c9449e3aa2b 2024-12-08T03:42:53,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/0005f069f22347e999d788508ff096bc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/0005f069f22347e999d788508ff096bc 2024-12-08T03:42:53,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/386656414f43401688ed8ff50e40f4db 2024-12-08T03:42:53,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/e50d8d941ad24c84b93312790fab3320 2024-12-08T03:42:53,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/5afeb18bd42841339ec440354957a7bc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/5afeb18bd42841339ec440354957a7bc 2024-12-08T03:42:53,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/80b726f8e752436cabddc3f835c01a16 2024-12-08T03:42:53,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f745bdb3a56445b98d92aab3a298d8fc 2024-12-08T03:42:53,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/34470c31b5f544a188321d0395d9b87a 2024-12-08T03:42:53,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/b1ea61ef6be84c31a1992f6b7e3bd87d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/c724aaacfc9d4ec28169e7a8e23b9508, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/230d3d586e204283ab3e9a3b0aa7c834, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3d9c39a84de94e3786a446e14c4a68f6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3e81a19e04b14793b09f97a87be36cdd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/41e12b3898b2443b90fb8f54ba2d1cc1, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/1843979fd1014fc18fe1058a503462e2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/e31c22b4cff24e009932996d5fe399a4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/bc411ca140bc46bbbd30093b5b9a3a75, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ed2b699572114b7ea90cb04dc0ca291d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a2338fb6508a421a8dcc61d91e93e85e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/7bf823fb9a41497fba439dcb7454c934, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a54c52c7cbfa46a29868921b14730dbd, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cf2e8937ed0f4fcdad395c48c40c566f, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/0dcc065db9524c1bbfacd6310b1431d4, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/739915060b0c4a078a566fd4502b4104, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ae07fb45bf094841816eda112dd46e2d, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/106c231a876a4f9ebc2a956296c4af30, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ea9e53c655db4edfa61027951588cea1] to archive 2024-12-08T03:42:53,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:42:53,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/b1ea61ef6be84c31a1992f6b7e3bd87d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/b1ea61ef6be84c31a1992f6b7e3bd87d 2024-12-08T03:42:53,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/c724aaacfc9d4ec28169e7a8e23b9508 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/c724aaacfc9d4ec28169e7a8e23b9508 2024-12-08T03:42:53,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/230d3d586e204283ab3e9a3b0aa7c834 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/230d3d586e204283ab3e9a3b0aa7c834 2024-12-08T03:42:53,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3d9c39a84de94e3786a446e14c4a68f6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3d9c39a84de94e3786a446e14c4a68f6 2024-12-08T03:42:53,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3e81a19e04b14793b09f97a87be36cdd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/3e81a19e04b14793b09f97a87be36cdd 2024-12-08T03:42:53,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/41e12b3898b2443b90fb8f54ba2d1cc1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/41e12b3898b2443b90fb8f54ba2d1cc1 2024-12-08T03:42:53,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/1843979fd1014fc18fe1058a503462e2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/1843979fd1014fc18fe1058a503462e2 2024-12-08T03:42:53,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/e31c22b4cff24e009932996d5fe399a4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/e31c22b4cff24e009932996d5fe399a4 2024-12-08T03:42:53,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/bc411ca140bc46bbbd30093b5b9a3a75 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/bc411ca140bc46bbbd30093b5b9a3a75 2024-12-08T03:42:53,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ed2b699572114b7ea90cb04dc0ca291d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ed2b699572114b7ea90cb04dc0ca291d 2024-12-08T03:42:53,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a2338fb6508a421a8dcc61d91e93e85e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a2338fb6508a421a8dcc61d91e93e85e 2024-12-08T03:42:53,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/7bf823fb9a41497fba439dcb7454c934 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/7bf823fb9a41497fba439dcb7454c934 2024-12-08T03:42:53,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a54c52c7cbfa46a29868921b14730dbd to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/a54c52c7cbfa46a29868921b14730dbd 2024-12-08T03:42:53,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cf2e8937ed0f4fcdad395c48c40c566f to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cf2e8937ed0f4fcdad395c48c40c566f 2024-12-08T03:42:53,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/0dcc065db9524c1bbfacd6310b1431d4 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/0dcc065db9524c1bbfacd6310b1431d4 2024-12-08T03:42:53,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/739915060b0c4a078a566fd4502b4104 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/739915060b0c4a078a566fd4502b4104 2024-12-08T03:42:53,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ae07fb45bf094841816eda112dd46e2d to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ae07fb45bf094841816eda112dd46e2d 2024-12-08T03:42:53,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/106c231a876a4f9ebc2a956296c4af30 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/106c231a876a4f9ebc2a956296c4af30 2024-12-08T03:42:53,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ea9e53c655db4edfa61027951588cea1 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/ea9e53c655db4edfa61027951588cea1 2024-12-08T03:42:53,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d006959255df4f0aaba2f650e046d59e, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1ab6879254314b3f8683ba5b8ab795ca, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/515a7cfdff1d4ac2bad012ccdf423696, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/947671ba0df34c0ca20b6d4c311b645b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/db41ec64d9e044109c3b0202f6e9c5da, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/254ac1523c0744aa97e25827982fbcc6, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/56bdadc88b4246beb413ad6c51ffe534, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/5232074511b44d8580798ba283096fee, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/23210f0b0fca4c55b232ba3d15de5392, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/3aa8b2a6f2534279b43d17f814851b04, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/79b9df35140f46a091f7e64abde5bea5, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/a8150ecf54d6432195ddb63f87299988, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1f6b2ed66139426f818e247da735ccb2, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/bb74d449bb5444a1ac44c14087cf6f9b, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d61d953aac9349a7b1980f69d1a39378, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/61fa52502bd24400907d4469f3f02d98, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/566b575020d242f28b4e27e2f4359b12, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/7a10d83f0fc540829c3a1674b6990d74] to archive 2024-12-08T03:42:53,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:42:53,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d006959255df4f0aaba2f650e046d59e to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d006959255df4f0aaba2f650e046d59e 2024-12-08T03:42:53,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/8d7208d77dd44b9eb16c1b6bb8a8d2f5 2024-12-08T03:42:53,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1ab6879254314b3f8683ba5b8ab795ca to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1ab6879254314b3f8683ba5b8ab795ca 2024-12-08T03:42:53,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/515a7cfdff1d4ac2bad012ccdf423696 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/515a7cfdff1d4ac2bad012ccdf423696 2024-12-08T03:42:53,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/947671ba0df34c0ca20b6d4c311b645b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/947671ba0df34c0ca20b6d4c311b645b 2024-12-08T03:42:53,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/db41ec64d9e044109c3b0202f6e9c5da to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/db41ec64d9e044109c3b0202f6e9c5da 2024-12-08T03:42:53,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/254ac1523c0744aa97e25827982fbcc6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/254ac1523c0744aa97e25827982fbcc6 2024-12-08T03:42:53,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/56bdadc88b4246beb413ad6c51ffe534 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/56bdadc88b4246beb413ad6c51ffe534 2024-12-08T03:42:53,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/5232074511b44d8580798ba283096fee to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/5232074511b44d8580798ba283096fee 2024-12-08T03:42:53,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/23210f0b0fca4c55b232ba3d15de5392 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/23210f0b0fca4c55b232ba3d15de5392 2024-12-08T03:42:53,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/3aa8b2a6f2534279b43d17f814851b04 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/3aa8b2a6f2534279b43d17f814851b04 2024-12-08T03:42:53,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/79b9df35140f46a091f7e64abde5bea5 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/79b9df35140f46a091f7e64abde5bea5 2024-12-08T03:42:53,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/a8150ecf54d6432195ddb63f87299988 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/a8150ecf54d6432195ddb63f87299988 2024-12-08T03:42:53,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1f6b2ed66139426f818e247da735ccb2 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/1f6b2ed66139426f818e247da735ccb2 2024-12-08T03:42:53,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/bb74d449bb5444a1ac44c14087cf6f9b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/bb74d449bb5444a1ac44c14087cf6f9b 2024-12-08T03:42:53,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d61d953aac9349a7b1980f69d1a39378 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d61d953aac9349a7b1980f69d1a39378 2024-12-08T03:42:53,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/61fa52502bd24400907d4469f3f02d98 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/61fa52502bd24400907d4469f3f02d98 2024-12-08T03:42:53,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/566b575020d242f28b4e27e2f4359b12 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/566b575020d242f28b4e27e2f4359b12 2024-12-08T03:42:53,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/7a10d83f0fc540829c3a1674b6990d74 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/7a10d83f0fc540829c3a1674b6990d74 2024-12-08T03:42:53,648 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-12-08T03:42:53,648 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. 2024-12-08T03:42:53,649 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for d51381d83515f3cce487395d9d692757: Waiting for close lock at 1733629371884Running coprocessor pre-close hooks at 1733629371884Disabling compacts and flushes for region at 1733629371884Disabling writes for close at 1733629371885 (+1 ms)Obtaining lock to block concurrent updates at 1733629371885Preparing flush snapshotting stores in d51381d83515f3cce487395d9d692757 at 1733629371885Finished memstore snapshotting TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733629371886 (+1 ms)Flushing stores of TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757. at 1733629371887 (+1 ms)Flushing d51381d83515f3cce487395d9d692757/A: creating writer at 1733629371887Flushing mob file d51381d83515f3cce487395d9d692757/A: appending metadata at 1733629371895 (+8 ms)Flushing mob file d51381d83515f3cce487395d9d692757/A: closing flushed file at 1733629371895Flushing d51381d83515f3cce487395d9d692757/A: appending metadata at 1733629372311 (+416 ms)Flushing d51381d83515f3cce487395d9d692757/A: closing flushed file at 1733629372311Flushing d51381d83515f3cce487395d9d692757/B: creating writer at 1733629372722 (+411 ms)Flushing d51381d83515f3cce487395d9d692757/B: appending metadata at 1733629372725 (+3 ms)Flushing d51381d83515f3cce487395d9d692757/B: closing flushed file at 1733629372725Flushing d51381d83515f3cce487395d9d692757/C: creating writer at 1733629373141 (+416 ms)Flushing d51381d83515f3cce487395d9d692757/C: appending metadata at 1733629373144 (+3 ms)Flushing d51381d83515f3cce487395d9d692757/C: closing flushed file at 1733629373144Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a17df25: reopening flushed file at 1733629373558 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eb1affe: reopening flushed file at 1733629373566 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60578b51: reopening flushed file at 1733629373571 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d51381d83515f3cce487395d9d692757 in 1692ms, sequenceid=304, compaction requested=true at 1733629373577 (+6 ms)Writing region close event to WAL at 1733629373644 (+67 ms)Running coprocessor post-close hooks at 1733629373648 (+4 ms)Closed at 1733629373648 2024-12-08T03:42:53,651 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,652 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=d51381d83515f3cce487395d9d692757, regionState=CLOSED 2024-12-08T03:42:53,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 because future has completed 2024-12-08T03:42:53,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-08T03:42:53,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure d51381d83515f3cce487395d9d692757, server=32df4b4bb439,42629,1733629194165 in 1.9230 sec 2024-12-08T03:42:53,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-08T03:42:53,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51381d83515f3cce487395d9d692757, UNASSIGN in 1.9280 sec 2024-12-08T03:42:53,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-08T03:42:53,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9330 sec 2024-12-08T03:42:53,664 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733629373664"}]},"ts":"1733629373664"} 2024-12-08T03:42:53,666 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:42:53,666 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:42:53,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9450 sec 2024-12-08T03:42:53,681 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T03:42:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T03:42:53,870 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:53,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:42:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:53,875 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T03:42:53,877 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:53,881 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C, FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/recovered.edits] 2024-12-08T03:42:53,889 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/39c8eff9b8f449948bc9bf626f7e5228 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/39c8eff9b8f449948bc9bf626f7e5228 2024-12-08T03:42:53,891 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/7667b18ca5414b30a867b297f810760b to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/7667b18ca5414b30a867b297f810760b 2024-12-08T03:42:53,894 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f89722237d4e410992326586d7b070d6 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/A/f89722237d4e410992326586d7b070d6 2024-12-08T03:42:53,898 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/321ea8ae048245b1b7c6365bb0440eee to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/321ea8ae048245b1b7c6365bb0440eee 2024-12-08T03:42:53,900 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/4e882d960daa463f9405811faf40e32c to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/4e882d960daa463f9405811faf40e32c 2024-12-08T03:42:53,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cffab11cfd6e494490a91d693d338d92 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/B/cffab11cfd6e494490a91d693d338d92 2024-12-08T03:42:53,905 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/92dc79f07905415d8177c20cefdee212 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/92dc79f07905415d8177c20cefdee212 2024-12-08T03:42:53,907 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/954f2810138f4a25b01244912727a306 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/954f2810138f4a25b01244912727a306 2024-12-08T03:42:53,908 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d5e9b6ffbc7245e386f7ed424df26933 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/C/d5e9b6ffbc7245e386f7ed424df26933 2024-12-08T03:42:53,911 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/recovered.edits/307.seqid to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757/recovered.edits/307.seqid 2024-12-08T03:42:53,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/default/TestAcidGuarantees/d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,912 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:42:53,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:42:53,913 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T03:42:53,917 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120811a0229edb4f4b2bb04ce43b3916ee3d_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120811a0229edb4f4b2bb04ce43b3916ee3d_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,918 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208120ad7657f6e4c02b1b1e473eea1ddd8_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208120ad7657f6e4c02b1b1e473eea1ddd8_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,920 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081c12409e1afc4ac28ac084b70e0eea5a_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081c12409e1afc4ac28ac084b70e0eea5a_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,922 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084776e7fdaefe4391a95d5c3b665a35d0_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084776e7fdaefe4391a95d5c3b665a35d0_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,924 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084afb6ea798dc4bd49985218557fc1234_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084afb6ea798dc4bd49985218557fc1234_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,926 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208597393d57d1f4d988bb1b6e7a31fff0f_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208597393d57d1f4d988bb1b6e7a31fff0f_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087515de1d9ca84094b04d1f9e65ea3bb6_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087515de1d9ca84094b04d1f9e65ea3bb6_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,932 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208817ce0e8f38a41a4b032b5f27dddc1de_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208817ce0e8f38a41a4b032b5f27dddc1de_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,934 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208878cd35fe6bd423ea9d0fa262cd07ae9_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208878cd35fe6bd423ea9d0fa262cd07ae9_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,936 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088e98f863791344c6b140e4a591c7efb2_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088e98f863791344c6b140e4a591c7efb2_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,938 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a3a8214a9dfd453c851365c621b84f8b_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a3a8214a9dfd453c851365c621b84f8b_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,940 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ad5272fbc995417abf2aa8c31ec14910_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ad5272fbc995417abf2aa8c31ec14910_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,941 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b90fe3e953de4a0981c8353659c06435_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b90fe3e953de4a0981c8353659c06435_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,943 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bee020c4c3924b2fa31e83701d61e724_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bee020c4c3924b2fa31e83701d61e724_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,944 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ed709810ac50409f88f6526499851f59_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ed709810ac50409f88f6526499851f59_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,946 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f868eb5ca8904da0be3ef68db139e1b6_d51381d83515f3cce487395d9d692757 to hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f868eb5ca8904da0be3ef68db139e1b6_d51381d83515f3cce487395d9d692757 2024-12-08T03:42:53,946 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:42:53,949 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:53,952 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:42:53,955 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:42:53,956 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:53,956 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:42:53,956 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733629373956"}]},"ts":"9223372036854775807"} 2024-12-08T03:42:53,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:42:53,958 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d51381d83515f3cce487395d9d692757, NAME => 'TestAcidGuarantees,,1733629345870.d51381d83515f3cce487395d9d692757.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:42:53,958 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:42:53,958 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733629373958"}]},"ts":"9223372036854775807"} 2024-12-08T03:42:53,960 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:42:53,960 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:42:53,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-12-08T03:42:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T03:42:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:42:53,989 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:42:53,995 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 236) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=286 (was 278) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17014 (was 17078) 2024-12-08T03:42:53,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T03:42:53,995 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:42:53,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:42:53,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:53,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:53,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:42:53,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T03:42:53,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-12-08T03:42:53,996 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32df4b4bb439,46215,1733629193261 2024-12-08T03:42:54,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T03:42:54,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T03:42:54,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:42:54,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:42:54,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T03:42:54,062 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:42:54,063 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:42:54,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:54,063 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T03:42:54,063 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T03:42:54,063 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,42629,1733629194165' ***** 2024-12-08T03:42:54,064 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T03:42:54,064 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T03:42:54,065 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T03:42:54,065 INFO [RS:0;32df4b4bb439:42629 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T03:42:54,065 INFO [RS:0;32df4b4bb439:42629 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T03:42:54,065 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,42629,1733629194165 2024-12-08T03:42:54,065 INFO [RS:0;32df4b4bb439:42629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T03:42:54,065 INFO [RS:0;32df4b4bb439:42629 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32df4b4bb439:42629. 2024-12-08T03:42:54,065 DEBUG [RS:0;32df4b4bb439:42629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:42:54,065 DEBUG [RS:0;32df4b4bb439:42629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:54,066 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T03:42:54,066 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T03:42:54,066 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T03:42:54,066 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T03:42:54,067 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T03:42:54,067 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T03:42:54,067 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T03:42:54,067 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:54,067 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T03:42:54,067 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T03:42:54,067 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T03:42:54,067 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T03:42:54,067 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-08T03:42:54,084 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/info/0ff0be9dc51b49aaa13e94b22536b4cf is 95, key is TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0./info:/1733629225591/DeleteFamily/seqid=0 2024-12-08T03:42:54,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742396_1572 (size=6514) 2024-12-08T03:42:54,145 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T03:42:54,267 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:54,467 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:54,489 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/info/0ff0be9dc51b49aaa13e94b22536b4cf 2024-12-08T03:42:54,516 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/ns/6c4efe5d312546c1a378cccf2ca24493 is 93, key is TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0./ns:/1733629225552/DeleteFamily/seqid=0 2024-12-08T03:42:54,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742397_1573 (size=5906) 2024-12-08T03:42:54,668 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:54,829 INFO [regionserver/32df4b4bb439:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T03:42:54,829 INFO [regionserver/32df4b4bb439:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T03:42:54,868 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:54,921 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/ns/6c4efe5d312546c1a378cccf2ca24493 2024-12-08T03:42:54,951 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/rep_barrier/52fca01eea5849b7b79b41b92f9b2f8d is 102, key is TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0./rep_barrier:/1733629225552/DeleteFamily/seqid=0 2024-12-08T03:42:54,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742398_1574 (size=6025) 2024-12-08T03:42:55,068 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:55,269 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:55,355 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/rep_barrier/52fca01eea5849b7b79b41b92f9b2f8d 2024-12-08T03:42:55,383 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/table/5af6b5da4a954a388f1c863fb3d1b873 is 96, key is TestAcidGuarantees,,1733629196849.9b6311f5dfb022964ba60e0ab8e515b0./table:/1733629225552/DeleteFamily/seqid=0 2024-12-08T03:42:55,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742399_1575 (size=5936) 2024-12-08T03:42:55,469 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:55,669 DEBUG [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T03:42:55,761 INFO [regionserver/32df4b4bb439:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-12-08T03:42:55,788 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/table/5af6b5da4a954a388f1c863fb3d1b873 2024-12-08T03:42:55,797 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/info/0ff0be9dc51b49aaa13e94b22536b4cf as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/info/0ff0be9dc51b49aaa13e94b22536b4cf 2024-12-08T03:42:55,801 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/info/0ff0be9dc51b49aaa13e94b22536b4cf, entries=12, sequenceid=90, filesize=6.4 K 2024-12-08T03:42:55,802 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/ns/6c4efe5d312546c1a378cccf2ca24493 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/ns/6c4efe5d312546c1a378cccf2ca24493 2024-12-08T03:42:55,806 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/ns/6c4efe5d312546c1a378cccf2ca24493, entries=8, sequenceid=90, filesize=5.8 K 2024-12-08T03:42:55,807 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/rep_barrier/52fca01eea5849b7b79b41b92f9b2f8d as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/rep_barrier/52fca01eea5849b7b79b41b92f9b2f8d 2024-12-08T03:42:55,809 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/rep_barrier/52fca01eea5849b7b79b41b92f9b2f8d, entries=6, sequenceid=90, filesize=5.9 K 2024-12-08T03:42:55,810 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/.tmp/table/5af6b5da4a954a388f1c863fb3d1b873 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/table/5af6b5da4a954a388f1c863fb3d1b873 2024-12-08T03:42:55,812 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/table/5af6b5da4a954a388f1c863fb3d1b873, entries=7, sequenceid=90, filesize=5.8 K 2024-12-08T03:42:55,813 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false 2024-12-08T03:42:55,819 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-08T03:42:55,819 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T03:42:55,819 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T03:42:55,819 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733629374067Running coprocessor pre-close hooks at 1733629374067Disabling compacts and flushes for region at 1733629374067Disabling writes for close at 1733629374067Obtaining lock to block concurrent updates at 1733629374067Preparing flush snapshotting stores in 1588230740 at 1733629374067Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733629374067Flushing stores of hbase:meta,,1.1588230740 at 1733629374068 (+1 ms)Flushing 1588230740/info: creating writer at 1733629374068Flushing 1588230740/info: appending metadata at 1733629374084 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733629374084Flushing 1588230740/ns: creating writer at 1733629374500 (+416 ms)Flushing 1588230740/ns: appending metadata at 1733629374516 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733629374516Flushing 1588230740/rep_barrier: creating writer at 1733629374929 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733629374951 (+22 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733629374951Flushing 1588230740/table: creating writer at 1733629375364 (+413 ms)Flushing 1588230740/table: appending metadata at 1733629375383 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733629375383Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aed003d: reopening flushed file at 1733629375796 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2b9c3a: reopening flushed file at 1733629375801 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49228abe: reopening flushed file at 1733629375806 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@439e0ab2: reopening flushed file at 1733629375810 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false at 1733629375813 (+3 ms)Writing region close event to WAL at 1733629375816 (+3 ms)Running coprocessor post-close hooks at 1733629375819 (+3 ms)Closed at 1733629375819 2024-12-08T03:42:55,819 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T03:42:55,870 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,42629,1733629194165; all regions closed. 2024-12-08T03:42:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741834_1010 (size=25344) 2024-12-08T03:42:55,879 DEBUG [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/oldWALs 2024-12-08T03:42:55,879 INFO [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C42629%2C1733629194165.meta:.meta(num 1733629196366) 2024-12-08T03:42:55,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741833_1009 (size=13245685) 2024-12-08T03:42:55,885 DEBUG [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/oldWALs 2024-12-08T03:42:55,885 INFO [RS:0;32df4b4bb439:42629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C42629%2C1733629194165:(num 1733629195908) 2024-12-08T03:42:55,885 DEBUG [RS:0;32df4b4bb439:42629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:42:55,885 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T03:42:55,886 INFO [RS:0;32df4b4bb439:42629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T03:42:55,886 INFO [RS:0;32df4b4bb439:42629 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T03:42:55,886 INFO [RS:0;32df4b4bb439:42629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T03:42:55,886 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T03:42:55,887 INFO [RS:0;32df4b4bb439:42629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42629 2024-12-08T03:42:55,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T03:42:55,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,42629,1733629194165 2024-12-08T03:42:55,967 INFO [RS:0;32df4b4bb439:42629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T03:42:55,969 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32df4b4bb439,42629,1733629194165] 2024-12-08T03:42:55,991 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32df4b4bb439,42629,1733629194165 already deleted, retry=false 2024-12-08T03:42:55,991 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32df4b4bb439,42629,1733629194165 expired; onlineServers=0 2024-12-08T03:42:55,991 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32df4b4bb439,46215,1733629193261' ***** 2024-12-08T03:42:55,991 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T03:42:55,992 INFO [M:0;32df4b4bb439:46215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T03:42:55,992 INFO [M:0;32df4b4bb439:46215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T03:42:55,992 DEBUG [M:0;32df4b4bb439:46215 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T03:42:55,992 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T03:42:55,992 DEBUG [M:0;32df4b4bb439:46215 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T03:42:55,992 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733629195478 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733629195478,5,FailOnTimeoutGroup] 2024-12-08T03:42:55,992 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733629195473 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733629195473,5,FailOnTimeoutGroup] 2024-12-08T03:42:55,992 INFO [M:0;32df4b4bb439:46215 {}] hbase.ChoreService(370): Chore service for: master/32df4b4bb439:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T03:42:55,992 INFO [M:0;32df4b4bb439:46215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T03:42:55,992 DEBUG [M:0;32df4b4bb439:46215 {}] master.HMaster(1795): Stopping service threads 2024-12-08T03:42:55,992 INFO [M:0;32df4b4bb439:46215 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T03:42:55,993 INFO [M:0;32df4b4bb439:46215 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T03:42:55,993 ERROR [M:0;32df4b4bb439:46215 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-08T03:42:55,994 INFO [M:0;32df4b4bb439:46215 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T03:42:55,994 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T03:42:55,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T03:42:55,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:42:56,000 DEBUG [M:0;32df4b4bb439:46215 {}] zookeeper.ZKUtil(347): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T03:42:56,000 WARN [M:0;32df4b4bb439:46215 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T03:42:56,000 INFO [M:0;32df4b4bb439:46215 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/.lastflushedseqids 2024-12-08T03:42:56,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742400_1576 (size=72) 2024-12-08T03:42:56,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T03:42:56,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42629-0x100038fe7ae0001, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T03:42:56,084 INFO [RS:0;32df4b4bb439:42629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T03:42:56,084 INFO [RS:0;32df4b4bb439:42629 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,42629,1733629194165; zookeeper connection closed. 2024-12-08T03:42:56,084 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@178ce89c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@178ce89c 2024-12-08T03:42:56,085 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T03:42:56,414 INFO [M:0;32df4b4bb439:46215 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T03:42:56,415 INFO [M:0;32df4b4bb439:46215 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T03:42:56,415 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T03:42:56,416 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:42:56,416 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:42:56,416 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T03:42:56,416 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:42:56,416 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=725.01 KB heapSize=888.21 KB 2024-12-08T03:42:56,437 DEBUG [M:0;32df4b4bb439:46215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86ad9b9cabff44f8aeee60c089496720 is 82, key is hbase:meta,,1/info:regioninfo/1733629196457/Put/seqid=0 2024-12-08T03:42:56,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742401_1577 (size=5672) 2024-12-08T03:42:56,842 INFO [M:0;32df4b4bb439:46215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2006 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86ad9b9cabff44f8aeee60c089496720 2024-12-08T03:42:56,870 DEBUG [M:0;32df4b4bb439:46215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a4458cdf7534413e97e8657bed5d2558 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733629349043/Put/seqid=0 2024-12-08T03:42:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742402_1578 (size=43161) 2024-12-08T03:42:57,276 INFO [M:0;32df4b4bb439:46215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=724.46 KB at sequenceid=2006 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a4458cdf7534413e97e8657bed5d2558 2024-12-08T03:42:57,282 INFO [M:0;32df4b4bb439:46215 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a4458cdf7534413e97e8657bed5d2558 2024-12-08T03:42:57,301 DEBUG [M:0;32df4b4bb439:46215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/698ff3a63aa94d4292b108d099e021d4 is 69, key is 32df4b4bb439,42629,1733629194165/rs:state/1733629195548/Put/seqid=0 2024-12-08T03:42:57,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073742403_1579 (size=5156) 2024-12-08T03:42:57,705 INFO [M:0;32df4b4bb439:46215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2006 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/698ff3a63aa94d4292b108d099e021d4 2024-12-08T03:42:57,717 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86ad9b9cabff44f8aeee60c089496720 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/86ad9b9cabff44f8aeee60c089496720 2024-12-08T03:42:57,721 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/86ad9b9cabff44f8aeee60c089496720, entries=8, sequenceid=2006, filesize=5.5 K 2024-12-08T03:42:57,722 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a4458cdf7534413e97e8657bed5d2558 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a4458cdf7534413e97e8657bed5d2558 2024-12-08T03:42:57,725 INFO [M:0;32df4b4bb439:46215 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a4458cdf7534413e97e8657bed5d2558 2024-12-08T03:42:57,725 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a4458cdf7534413e97e8657bed5d2558, entries=164, sequenceid=2006, filesize=42.1 K 2024-12-08T03:42:57,726 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/698ff3a63aa94d4292b108d099e021d4 as hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/698ff3a63aa94d4292b108d099e021d4 2024-12-08T03:42:57,729 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/9a8af5c0-9308-2803-4bfc-dbdcc9dac81a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/698ff3a63aa94d4292b108d099e021d4, entries=1, sequenceid=2006, filesize=5.0 K 2024-12-08T03:42:57,730 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(3140): Finished flush of dataSize ~725.01 KB/742412, heapSize ~887.91 KB/909224, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2006, compaction requested=false 2024-12-08T03:42:57,731 INFO [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:42:57,731 DEBUG [M:0;32df4b4bb439:46215 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733629376415Disabling compacts and flushes for region at 1733629376415Disabling writes for close at 1733629376416 (+1 ms)Obtaining lock to block concurrent updates at 1733629376416Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733629376416Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=742412, getHeapSize=909464, getOffHeapSize=0, getCellsCount=2195 at 1733629376417 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733629376418 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733629376418Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733629376437 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733629376437Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733629376851 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733629376870 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733629376870Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733629377283 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733629377300 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733629377300Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d940354: reopening flushed file at 1733629377715 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e55d1cb: reopening flushed file at 1733629377722 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@212f8566: reopening flushed file at 1733629377725 (+3 ms)Finished flush of dataSize ~725.01 KB/742412, heapSize ~887.91 KB/909224, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2006, compaction requested=false at 1733629377730 (+5 ms)Writing region close event to WAL at 1733629377731 (+1 ms)Closed at 1733629377731 2024-12-08T03:42:57,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741830_1006 (size=873977) 2024-12-08T03:42:57,734 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T03:42:57,734 INFO [M:0;32df4b4bb439:46215 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T03:42:57,734 INFO [M:0;32df4b4bb439:46215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46215 2024-12-08T03:42:57,734 INFO [M:0;32df4b4bb439:46215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T03:42:57,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T03:42:57,883 INFO [M:0;32df4b4bb439:46215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T03:42:57,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46215-0x100038fe7ae0000, quorum=127.0.0.1:53034, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T03:42:57,889 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T03:42:57,892 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T03:42:57,892 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T03:42:57,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T03:42:57,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/hadoop.log.dir/,STOPPED} 2024-12-08T03:42:57,895 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T03:42:57,895 WARN [BP-1611339395-172.17.0.2-1733629189334 heartbeating to localhost/127.0.0.1:41611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T03:42:57,895 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T03:42:57,895 WARN [BP-1611339395-172.17.0.2-1733629189334 heartbeating to localhost/127.0.0.1:41611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1611339395-172.17.0.2-1733629189334 (Datanode Uuid cea92d07-14e2-464c-89b3-db1f91e631d3) service to localhost/127.0.0.1:41611 2024-12-08T03:42:57,897 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/data/data1/current/BP-1611339395-172.17.0.2-1733629189334 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T03:42:57,897 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/cluster_73a40b8f-afdc-9b7e-daa5-855b9de847e6/data/data2/current/BP-1611339395-172.17.0.2-1733629189334 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T03:42:57,898 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T03:42:57,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T03:42:57,909 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T03:42:57,909 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T03:42:57,909 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T03:42:57,909 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/65138b0c-d475-9a1b-83c5-1d8f27db4e0f/hadoop.log.dir/,STOPPED} 2024-12-08T03:42:57,921 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T03:42:58,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down